From 262a1278235649aada1a78cec4d52a4ff6ea646b Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 14 Oct 2025 10:55:59 -0400 Subject: [PATCH 1/2] Update SmartscraperDescription.ts --- nodes/MarkdownifyDescription.ts | 21 ++++++++++ nodes/ScrapegraphAI/ScrapegraphAi.node.ts | 47 +++++++++++++++++++++++ nodes/SearchscraperDescription.ts | 35 +++++++++++++++++ nodes/SmartscraperDescription.ts | 22 +++++++++++ 4 files changed, 125 insertions(+) diff --git a/nodes/MarkdownifyDescription.ts b/nodes/MarkdownifyDescription.ts index 3dbbfd7..0160d78 100644 --- a/nodes/MarkdownifyDescription.ts +++ b/nodes/MarkdownifyDescription.ts @@ -18,12 +18,33 @@ export const markdownifyOperations: INodeProperties[] = [ action: 'Convert a webpage or article to clean markdown useful for blogs dev docs and more', description: 'Convert a webpage or article to clean markdown – useful for blogs, dev docs and more', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a conversion', + description: 'Retrieve the status and results of a markdown conversion by request ID', + }, ], default: 'convert', }, ]; export const markdownifyFields: INodeProperties[] = [ + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a convert operation', + displayOptions: { + show: { + resource: ['markdownify'], + operation: ['getStatus'], + }, + }, + }, + // Fields for convert operation { displayName: 'Website URL', name: 'websiteUrl', diff --git a/nodes/ScrapegraphAI/ScrapegraphAi.node.ts b/nodes/ScrapegraphAI/ScrapegraphAi.node.ts index 51291e6..f6b1a8b 100644 --- a/nodes/ScrapegraphAI/ScrapegraphAi.node.ts +++ b/nodes/ScrapegraphAI/ScrapegraphAi.node.ts @@ -148,12 +148,28 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/smartscraper/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'searchscraper') { if (operation === 'search') { const userPrompt = this.getNodeParameter('userPrompt', i) as string; const numResults = this.getNodeParameter('numResults', i) as number; + const extractionMode = this.getNodeParameter('extractionMode', i, true) as boolean; const enableScrolling = this.getNodeParameter('enableScrolling', i, false) as boolean; const enablePagination = this.getNodeParameter('enablePagination', i, false) as boolean; const useOutputSchema = this.getNodeParameter('useOutputSchema', i, false) as boolean; @@ -161,6 +177,7 @@ export class ScrapegraphAi implements INodeType { const requestBody: any = { user_prompt: userPrompt, num_results: numResults, + extraction_mode: extractionMode, }; // Add number_of_scrolls if scrolling is enabled @@ -202,6 +219,21 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/searchscraper/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'smartcrawler') { @@ -286,6 +318,21 @@ export class ScrapegraphAi implements INodeType { returnData.push({ json: response, pairedItem: { item: i } }); } + + if (operation === 'getStatus') { + const requestId = this.getNodeParameter('requestId', i) as string; + + const response = await this.helpers.httpRequestWithAuthentication.call(this, 'scrapegraphAIApi', { + method: 'GET', + url: `${baseUrl}/markdownify/${requestId}`, + headers: { + 'Accept': 'application/json', + }, + json: true, + }); + + returnData.push({ json: response, pairedItem: { item: i } }); + } } if (resource === 'scrape') { diff --git a/nodes/SearchscraperDescription.ts b/nodes/SearchscraperDescription.ts index 1a67aac..61f4a9a 100644 --- a/nodes/SearchscraperDescription.ts +++ b/nodes/SearchscraperDescription.ts @@ -18,12 +18,34 @@ export const searchscraperOperations: INodeProperties[] = [ action: 'Perform ai powered site wide search and structured data extraction ideal for knowledge retrieval', description: 'Perform AI-powered site-wide search and structured data extraction – ideal for knowledge retrieval', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a search operation', + description: 'Retrieve the status and results of a search scraping operation by request ID', + }, ], default: 'search', }, ]; export const searchscraperFields: INodeProperties[] = [ + // Fields for getStatus operation + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a search operation', + displayOptions: { + show: { + resource: ['searchscraper'], + operation: ['getStatus'], + }, + }, + }, + // Fields for search operation { displayName: 'User Prompt', name: 'userPrompt', @@ -97,6 +119,19 @@ export const searchscraperFields: INodeProperties[] = [ }, ], }, + { + displayName: 'Extraction Mode', + name: 'extractionMode', + type: 'boolean', + default: true, + description: 'Whether to use AI extraction (true) for structured data or markdown conversion (false) for raw content', + displayOptions: { + show: { + resource: ['searchscraper'], + operation: ['search'], + }, + }, + }, { displayName: 'Enable Infinite Scrolling', name: 'enableScrolling', diff --git a/nodes/SmartscraperDescription.ts b/nodes/SmartscraperDescription.ts index 86d6ff2..bea7a06 100644 --- a/nodes/SmartscraperDescription.ts +++ b/nodes/SmartscraperDescription.ts @@ -18,12 +18,34 @@ export const smartscraperOperations: INodeProperties[] = [ action: 'Autonomously extract live data from any website perfect for e commerce job boards lead capture and more', description: 'Autonomously extract live data from any website – perfect for e-commerce, job boards, lead capture and more', }, + { + name: 'Get Status', + value: 'getStatus', + action: 'Get the status and results of a scraping operation', + description: 'Retrieve the status and results of a scraping operation by request ID', + }, ], default: 'scrape', }, ]; export const smartscraperFields: INodeProperties[] = [ + // Fields for getStatus operation + { + displayName: 'Request ID', + name: 'requestId', + type: 'string', + required: true, + default: '', + description: 'The request ID returned from a scrape operation', + displayOptions: { + show: { + resource: ['smartscraper'], + operation: ['getStatus'], + }, + }, + }, + // Fields for scrape operation { displayName: 'Website URL', name: 'websiteUrl', From 4f7b79b13adf0eddf2b375ca774bab6351459d72 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Tue, 14 Oct 2025 11:40:46 -0400 Subject: [PATCH 2/2] Update MarkdownifyDescription.ts --- nodes/MarkdownifyDescription.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/MarkdownifyDescription.ts b/nodes/MarkdownifyDescription.ts index 0160d78..d9aa90e 100644 --- a/nodes/MarkdownifyDescription.ts +++ b/nodes/MarkdownifyDescription.ts @@ -30,6 +30,7 @@ export const markdownifyOperations: INodeProperties[] = [ ]; export const markdownifyFields: INodeProperties[] = [ + // Fields for getStatus operation { displayName: 'Request ID', name: 'requestId',