Quantcast
Channel: Active questions tagged rest - Stack Overflow
Viewing all articles
Browse latest Browse all 3619

SISS package not extracting all records from Api to database

$
0
0

I am trying to move data from Json Rest API to my database. I am using the data flow task with script task as source and OLE DB as destination.

Sample data

{"Id": 1,"Name": "Project A","owner": "User A","status": "Active","listedItems": [        {"column1": "100", "column2": "1000", "column3": "Type A"},        {"column1": "200", "column2": "2000", "column3": "Type B"},        {"column1": "300", "column2": "3000", "column3": "Type C"}    ]}

I want to move only the column1 , column2 and column3 data into a database.

    using System;    using System.Data;    using Microsoft.SqlServer.Dts.Pipeline.Wrapper;    using Microsoft.SqlServer.Dts.Runtime.Wrapper;    using System.Collections.Generic;    using System.IO;    using System.Net;    using static System.Windows.Forms.VisualStyles.VisualStyleElement.Tab;    using System.Text.RegularExpressions;    public class ScriptMain : UserComponent     {         public override void CreateNewOutputRows()             {             // Specify the security protocol             System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;             // Define the API URL             string apiUrl = "URL"; // Update with your actual API URL             string apiKey = "apiKey";             try             {                 HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl);                 request.Method = "GET";                 request.Headers["Authorization"] = "Bearer " + apiKey;                 using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())                 {                     using (StreamReader reader = new StreamReader(response.GetResponseStream()))                     {                         string jsonResponse = reader.ReadToEnd();                         ParseInventoryItems(jsonResponse);                     }                 }             }             catch (Exception ex)             {                 System.Diagnostics.Debug.WriteLine("Error: " + ex.Message);             }         }         private void ParseInventoryItems(string jsonResponse)         {             try             {                 // Extract inventory items as before                 string pattern = "\"listedItems\":\\[(.*?)\\]";                 Match match = Regex.Match(jsonResponse, pattern);                 if (match.Success)                 {                     string listedItemsContent = match.Groups[1].Value;                     var items = listedItemsContent.Split(new string[] { "},{" }, StringSplitOptions.None);                     int rowCount = 0;  // Count of rows added                     foreach (var item in items)                     {                         string cleanedItem = item.Replace("{", "").Replace("}", "").Trim();                         var keyValuePairs = cleanedItem.Split(',');                         Output0Buffer.AddRow();  // Add a new row                         // Initialize values                         Output0Buffer.column1 = string.Empty;                         Output0Buffer.column2 = string.Empty;                         Output0Buffer.column3 = string.Empty;                         // Parse key-value pairs                         foreach (var pair in keyValuePairs)                         {                             var keyValue = pair.Split(':');                             if (keyValue.Length != 2) continue;                             string key = keyValue[0].Trim().Trim('"');                             string value = keyValue[1].Trim().Trim('"');                             // Set output buffer values                             if (key.Equals("column1", StringComparison.OrdinalIgnoreCase))                             {                                 Output0Buffer.column1 = value;                             }                             else if (key.Equals("column2", StringComparison.OrdinalIgnoreCase))                             {                                 Output0Buffer.column2 = value;                             }                             else if (key.Equals("tcolum3", StringComparison.OrdinalIgnoreCase))                             {                                 Output0Buffer.column3 = value;                             }                         }                         rowCount++;  // Increment row count                                 // Log the current row information                     }                     // Final log of total rows processed                     System.Diagnostics.Debug.WriteLine($"Total Rows Processed: {rowCount}");                 }                 else                 {                     System.Diagnostics.Debug.WriteLine("No listedItems found in JSON response.");                 }             }             catch (Exception ex)             {                 System.Diagnostics.Debug.WriteLine("Error parsing inventory items: " + ex.Message);             }         }     }

The above code is only extracting the 1st item in list ("column1": "100", "column2": "1000", "column3": "Type A") How can I get all the items in that list?


Viewing all articles
Browse latest Browse all 3619

Trending Articles



<script src="https://jsc.adskeeper.com/r/s/rssing.com.1596347.js" async> </script>