Logo Questions Linux Laravel Mysql Ubuntu Git Menu
 

Optimizing XML import in SQL Server

I have this code to import a XML with 50mb, that contains around 26.280 entities. But this is taking to long, is with 50 minutes and still running, is that ok? or this code can be optimizing ?

INSERT INTO OSUSR_DFP_PEP_ENTITIES (ENT_ID, NAME, FIRSTNAME, LASTNAME, PREFIX, SUFFIX,
                      AKA, NAMESOURCE, PARENTID, GOVDESIGNATION, ENTRYTYPE, ENTRYCATEGORY, ENTRYSUBCATEGORY,
                      ORGANIZATION, POSITIONS, REMARKS, DOB, POB, COUNTRY, EXPIRATIONDATE, EFFECTIVEDATE, 
                      PICTUREFILE, LINKEDTO, RELATED_ID, SOURCEWEBLINK, TOUCHDATE, DIRECTID, PASSPORTID, 
                      NATIONALID, OTHERID, DOB2, ENTLEVEL, MASTERID, WATCH, RELATIONSHIPS) 
SELECT X.product.query('Ent_ID').value('.', 'INT'),
       X.product.query('Name').value('.', 'nvarchar(1000)'),
       X.product.query('FirstName').value('.', 'nvarchar(500)'),
       X.product.query('LastName').value('.', 'nvarchar(500)'),
       X.product.query('Prefix').value('.', 'nvarchar(500)'),
       X.product.query('Suffix').value('.', 'nvarchar(500)'),

       X.product.query('Aka').value('.', 'nvarchar(500)'),
       X.product.query('NameSource').value('.', 'nvarchar(500)'),
       X.product.query('ParentID').value('.', 'INT'),
       X.product.query('GovDesignation').value('.', 'nvarchar(500)'),
       X.product.query('EntryType').value('.', 'nvarchar(500)'),
       X.product.query('EntryCategory').value('.', 'nvarchar(500)'),
       X.product.query('EntrySubCategory').value('.', 'nvarchar(500)'),

       X.product.query('Organization').value('.', 'nvarchar(500)'),
       X.product.query('Positions').value('.', 'nvarchar(500)'),
       X.product.query('Remarks').value('.', 'nvarchar(max)'),
       X.product.query('DOB').value('.', 'nvarchar(500)'),
       X.product.query('POB').value('.', 'nvarchar(500)'),
       X.product.query('Country').value('.', 'nvarchar(500)'),
       X.product.query('ExpirationDate').value('.', 'nvarchar(500)'),
       X.product.query('EffectiveDate').value('.', 'nvarchar(500)'),

       X.product.query('PictureFile').value('.', 'nvarchar(500)'),
       X.product.query('LinkedTo').value('.', 'nvarchar(500)'),
       X.product.query('Related_ID').value('.', 'INT'),
       X.product.query('SourceWebLink').value('.', 'nvarchar(max)'),
       X.product.query('TouchDate').value('.', 'nvarchar(500)'),
       X.product.query('DirectID').value('.', 'nvarchar(500)'),
       X.product.query('PassportID').value('.', 'nvarchar(500)'),

       X.product.query('NationalID').value('.', 'nvarchar(500)'),
       X.product.query('OtherID').value('.', 'nvarchar(50)'),
       X.product.query('DOB2').value('.', 'nvarchar(500)'),
       X.product.query('EntLevel').value('.', 'nvarchar(500)'),
       X.product.query('MasterID').value('.', 'int'),
       X.product.query('Watch').value('.', 'bit'),
       X.product.query('Relationships').value('.', 'bit')
FROM ( 
SELECT CAST(x AS XML)
FROM OPENROWSET(
     BULK 'C:\temp\teste.xml',
     SINGLE_BLOB) AS T(x)
     ) AS T(x)
CROSS APPLY x.nodes('Export/Entities') AS X(product);  

--- Working version, only takes 42 seconds

DECLARE @X XML

SELECT @X = CAST(x AS XML)
FROM OPENROWSET(
     BULK 'C:\temp\teste.xml',
     SINGLE_BLOB) AS T(x)


INSERT INTO OSUSR_DFP_PEP_ENTITIES (ENT_ID, NAME, FIRSTNAME, LASTNAME, PREFIX, SUFFIX,
                      AKA, NAMESOURCE, PARENTID, GOVDESIGNATION, ENTRYTYPE, ENTRYCATEGORY, ENTRYSUBCATEGORY,
                      ORGANIZATION, POSITIONS, REMARKS, DOB, POB, COUNTRY, EXPIRATIONDATE, EFFECTIVEDATE, 
                      PICTUREFILE, LINKEDTO, RELATED_ID, SOURCEWEBLINK, TOUCHDATE, DIRECTID, PASSPORTID, 
                      NATIONALID, OTHERID, DOB2, ENTLEVEL, MASTERID, WATCH, RELATIONSHIPS) 
SELECT X.product.value('(Ent_ID/text())[1]', 'INT'),
       X.product.value('(Name/text())[1]', 'nvarchar(1000)'),
       X.product.value('(FirstName/text())[1]', 'nvarchar(500)'),
       X.product.value('(LastName/text())[1]', 'nvarchar(500)'),
       X.product.value('(Prefix/text())[1]', 'nvarchar(500)'),
       X.product.value('(Suffix/text())[1]', 'nvarchar(500)'),

       X.product.value('(Aka/text())[1]', 'nvarchar(500)'),
       X.product.value('(NameSource/text())[1]', 'nvarchar(500)'),
       X.product.value('(ParentID/text())[1]', 'INT'),
       X.product.value('(GovDesignation/text())[1]', 'nvarchar(500)'),
       X.product.value('(EntryType/text())[1]', 'nvarchar(500)'),
       X.product.value('(EntryCategory/text())[1]', 'nvarchar(500)'),
       X.product.value('(EntrySubCategory/text())[1]', 'nvarchar(500)'),

       X.product.value('(Organization/text())[1]', 'nvarchar(500)'),
       X.product.value('(Positions/text())[1]', 'nvarchar(500)'),
       X.product.value('(Remarks/text())[1]', 'nvarchar(max)'),
       X.product.value('(DOB/text())[1]', 'nvarchar(500)'),
       X.product.value('(POB/text())[1]', 'nvarchar(500)'),
       X.product.value('(Country/text())[1]', 'nvarchar(500)'),
       X.product.value('(ExpirationDate/text())[1]', 'nvarchar(500)'),
       X.product.value('(EffectiveDate/text())[1]', 'nvarchar(500)'),

       X.product.value('(PictureFile/text())[1]', 'nvarchar(500)'),
       X.product.value('(LinkedTo/text())[1]', 'nvarchar(500)'),
       X.product.value('(Related_ID/text())[1]', 'INT'),
       X.product.value('(SourceWebLink/text())[1]', 'nvarchar(max)'),
       X.product.value('(TouchDate/text())[1]', 'nvarchar(500)'),
       X.product.value('(DirectID/text())[1]', 'nvarchar(500)'),
       X.product.value('(PassportID/text())[1]', 'nvarchar(500)'),

       X.product.value('(NationalID/text())[1]', 'nvarchar(500)'),
       X.product.value('(OtherID/text())[1]', 'nvarchar(50)'),
       X.product.value('(DOB2/text())[1]', 'nvarchar(500)'),
       X.product.value('(EntLevel/text())[1]', 'nvarchar(500)'),
       X.product.value('(MasterID/text())[1]', 'INT'),
       X.product.value('(Watch/text())[1]'), 'bit',
       X.product.value('(Relationships/text())[1]', 'bit')

FROM @X.nodes('Export/Entities') AS X(product);  
like image 685
Luis Avatar asked Mar 25 '26 07:03

Luis


1 Answers

I suggest two things.

  1. Load the XML to a XML variable and use the XML variable in the query.
  2. Change the code where you extract the values to only use .value and use text().

Try this:

DECLARE @X XML

SELECT @X = CAST(x AS XML)
FROM OPENROWSET(
     BULK 'C:\temp\teste.xml',
     SINGLE_BLOB) AS T(x)

INSERT INTO OSUSR_DFP_PEP_ENTITIES
.
.
.
SELECT X.product.value('(Ent_ID/text())[1]'), 'INT'),
       X.product.value('(Name/text())[1]', 'nvarchar(1000)'),
.
.
.
FROM @X.nodes('Export/Entities') AS X(product);  
like image 125
Mikael Eriksson Avatar answered Mar 26 '26 23:03

Mikael Eriksson



Donate For Us

If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!