{
  "version": "1.0.0",
  "exported_at": "2026-06-03T16:45:00.000Z",
  "project": {
    "name": "Universal Content Scraper",
    "description": "Best-effort Bloomberg-compatible universal content scraper. Visits each URL in the batch URL list, scrolls for dynamically loaded content, extracts one structured row per page with url, title, cleaned JSON-wrapped content, author, published_at, format, and error_message. Bloomberg may show anti-bot, CAPTCHA, subscription, paywall, 403, or 404 pages; this template flags those conditions in error_message and leaves content empty for blocked/error pages. Pagination/navigation is implemented as a multi-URL batch loop using navigate.urls plus loop-continue; add valid article URLs to scrape all desired pages.",
    "color": "bg-[#4589ff]",
    "template_id": "ai-generated"
  },
  "blocks": [
    {
      "block_id": "set-window-size-1",
      "block_type": "process",
      "title": "Set Window Size",
      "description": "Set browser window dimensions",
      "position_x": 120,
      "position_y": 220,
      "config": {
        "width": 1920,
        "height": 1080,
        "color": "bg-[#4589ff]"
      }
    },
    {
      "block_id": "navigate-1",
      "block_type": "process",
      "title": "Navigate",
      "description": "Go to a URL",
      "position_x": 456,
      "position_y": 220,
      "config": {
        "urls": [
          "https://www.bloomberg.com/opinion/articles/"
        ],
        "color": "bg-[#08bdba]"
      }
    },
    {
      "block_id": "wait-for-page-load-1",
      "block_type": "process",
      "title": "Wait for Page Load",
      "description": "Wait for page to finish loading",
      "position_x": 792,
      "position_y": 220,
      "config": {
        "timeout": 30,
        "color": "bg-[#08bdba]"
      }
    },
    {
      "block_id": "wait-for-element-1",
      "block_type": "process",
      "title": "Wait for Element",
      "description": "Wait until element appears",
      "position_x": 1128,
      "position_y": 220,
      "config": {
        "selector": "body",
        "timeout": 30,
        "visible": true,
        "color": "bg-[#08bdba]"
      }
    },
    {
      "block_id": "inject-javascript-1",
      "block_type": "process",
      "title": "Inject JavaScript",
      "description": "Run custom JavaScript on the page",
      "position_x": 1464,
      "position_y": 220,
      "config": {
        "jsCode": "window.scrollTo(0, document.body.scrollHeight);",
        "waitForCompletion": true,
        "timeout": 10,
        "color": "bg-[#a56eff]"
      }
    },
    {
      "block_id": "sleep-1",
      "block_type": "process",
      "title": "Sleep",
      "description": "Wait for specified time",
      "position_x": 1800,
      "position_y": 220,
      "config": {
        "duration": 2,
        "color": "bg-[#a56eff]"
      }
    },
    {
      "block_id": "structured-export-1",
      "block_type": "process",
      "title": "Structured Export",
      "description": "Export data with custom columns",
      "position_x": 2136,
      "position_y": 220,
      "config": {
        "rowSelector": "body",
        "fileName": "universal-content.csv",
        "saveLocation": "C:\\Users\\theskd\\Documents\\UScraper\\templates",
        "includeHeaders": true,
        "fileMode": "append",
        "color": "bg-[#42be65]",
        "columns": [
          {
            "name": "url",
            "selector": "(() => location.href)()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "title",
            "selector": "(() => { const meta = document.querySelector(`meta[property='og:title'], meta[name='twitter:title']`); return (meta?.content || document.querySelector('h1')?.innerText || document.title || '').trim(); })()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "content",
            "selector": "(() => { const hay = `${document.title} ${document.body.innerText.slice(0, 6000)}`; const isError = /unusual activity|not a robot|click the box below|captcha|browser supports JavaScript and cookies|reference ID below|403|Forbidden|Access Denied|Access to this page has been denied|404|Page Not Found|unable to find the page|Our apologies|sign in to continue|subscribe to continue|subscription required|Get the most important global markets news at your fingertips with a Bloomberg\\.com subscription|paywall/i.test(hay); if (isError) return JSON.stringify({ text: '' }); const source = document.querySelector('article') || document.querySelector('main') || document.querySelector(`[role='main']`) || ROW; const clone = source.cloneNode(true); clone.querySelectorAll(`script,style,noscript,nav,header,footer,aside,iframe,form,button,svg,[aria-hidden='true'],.advertisement,.ad,.ads,.paywall`).forEach(e => e.remove()); const parts = Array.from(clone.querySelectorAll('h1,h2,h3,p,li,blockquote')).map(e => e.innerText.trim()).filter(t => t.length > 20); const text = (parts.length ? parts.join('\\n\\n') : clone.innerText).replace(/\\s+\\n/g, '\\n').replace(/\\n{3,}/g, '\\n\\n').trim(); return JSON.stringify({ text }); })()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "author",
            "selector": "(() => { const hay = `${document.title} ${document.body.innerText.slice(0, 6000)}`; if (/unusual activity|not a robot|click the box below|captcha|403|Forbidden|Access Denied|404|Page Not Found|unable to find the page|sign in to continue|subscribe to continue|subscription required|paywall/i.test(hay)) return ''; const meta = document.querySelector(`meta[name='author'], meta[property='article:author'], meta[name='parsely-author']`); if (meta?.content) return meta.content.trim(); const ld = Array.from(document.querySelectorAll(`script[type='application/ld+json']`)).map(s => { try { return JSON.parse(s.textContent); } catch(e) { return null; } }).filter(Boolean).flatMap(x => Array.isArray(x) ? x : [x]); const a = ld.map(x => x.author).find(Boolean); if (Array.isArray(a)) return a.map(v => v.name || v).join(', '); if (a && typeof a === 'object') return a.name || ''; return (a || document.querySelector(`[rel='author'], .author, [class*='author' i]`)?.innerText || '').trim(); })()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "published_at",
            "selector": "(() => { const hay = `${document.title} ${document.body.innerText.slice(0, 6000)}`; if (/unusual activity|not a robot|click the box below|captcha|403|Forbidden|Access Denied|404|Page Not Found|unable to find the page|sign in to continue|subscribe to continue|subscription required|paywall/i.test(hay)) return ''; const meta = document.querySelector(`meta[property='article:published_time'], meta[name='article:published_time'], meta[name='parsely-pub-date'], meta[name='date'], meta[property='og:article:published_time']`); if (meta?.content) return meta.content.trim(); const time = document.querySelector('time[datetime]'); if (time?.getAttribute('datetime')) return time.getAttribute('datetime').trim(); const ld = Array.from(document.querySelectorAll(`script[type='application/ld+json']`)).map(s => { try { return JSON.parse(s.textContent); } catch(e) { return null; } }).filter(Boolean).flatMap(x => Array.isArray(x) ? x : [x]); return (ld.map(x => x.datePublished || x.dateCreated || x.uploadDate).find(Boolean) || '').trim(); })()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "format",
            "selector": "(() => 'json')()",
            "attribute": "text",
            "isJs": true
          },
          {
            "name": "error_message",
            "selector": "(() => { const hay = `${document.title} ${document.body.innerText.slice(0, 6000)}`; if (/unusual activity|not a robot|click the box below|captcha|browser supports JavaScript and cookies|reference ID below/i.test(hay)) return 'Anti-bot or CAPTCHA challenge detected'; if (/403|Forbidden|Access Denied|Access to this page has been denied/i.test(hay)) return 'Access blocked or 403 forbidden'; if (/404|Page Not Found|unable to find the page|Our apologies/i.test(hay)) return '404 page not found'; if (/sign in to continue|subscribe to continue|subscription required|Get the most important global markets news at your fingertips with a Bloomberg\\.com subscription|paywall/i.test(hay)) return 'Possible paywall, subscription, or login requirement'; return ''; })()",
            "attribute": "text",
            "isJs": true
          }
        ]
      }
    },
    {
      "block_id": "loop-continue-1",
      "block_type": "process",
      "title": "Loop Continue",
      "description": "Continue multi-input loop",
      "position_x": 2472,
      "position_y": 220,
      "config": {
        "color": "bg-[#8d8d8d]"
      }
    }
  ],
  "connections": [
    {
      "from_block_id": "set-window-size-1",
      "from_connector_id": "right",
      "to_block_id": "navigate-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "navigate-1",
      "from_connector_id": "right",
      "to_block_id": "wait-for-page-load-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "wait-for-page-load-1",
      "from_connector_id": "right",
      "to_block_id": "wait-for-element-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "wait-for-element-1",
      "from_connector_id": "right",
      "to_block_id": "inject-javascript-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "inject-javascript-1",
      "from_connector_id": "right",
      "to_block_id": "sleep-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "sleep-1",
      "from_connector_id": "right",
      "to_block_id": "structured-export-1",
      "to_connector_id": "left"
    },
    {
      "from_block_id": "structured-export-1",
      "from_connector_id": "right",
      "to_block_id": "loop-continue-1",
      "to_connector_id": "left"
    }
  ],
  "canvas_elements": [
    {
      "id": "group-entry",
      "element_type": "group",
      "title": "Entry & Setup",
      "color": "#4589ff",
      "position_x": 48,
      "position_y": 116,
      "width": 380,
      "height": 296,
      "z_index": 20,
      "data": {
        "memberBlockIds": [
          "set-window-size-1"
        ]
      }
    },
    {
      "id": "group-load",
      "element_type": "group",
      "title": "Page Load",
      "color": "#08bdba",
      "position_x": 384,
      "position_y": 116,
      "width": 1664,
      "height": 296,
      "z_index": 20,
      "data": {
        "memberBlockIds": [
          "navigate-1",
          "wait-for-page-load-1",
          "wait-for-element-1",
          "sleep-1"
        ]
      }
    },
    {
      "id": "group-interaction",
      "element_type": "group",
      "title": "Interaction",
      "color": "#a56eff",
      "position_x": 1392,
      "position_y": 116,
      "width": 380,
      "height": 296,
      "z_index": 20,
      "data": {
        "memberBlockIds": [
          "inject-javascript-1"
        ]
      }
    },
    {
      "id": "group-extract",
      "element_type": "group",
      "title": "Data Extraction",
      "color": "#42be65",
      "position_x": 2064,
      "position_y": 116,
      "width": 380,
      "height": 296,
      "z_index": 20,
      "data": {
        "memberBlockIds": [
          "structured-export-1"
        ]
      }
    },
    {
      "id": "group-pagination",
      "element_type": "group",
      "title": "Pagination Loop",
      "color": "#ff832b",
      "position_x": 2400,
      "position_y": 116,
      "width": 380,
      "height": 296,
      "z_index": 20,
      "data": {
        "memberBlockIds": [
          "loop-continue-1"
        ]
      }
    },
    {
      "id": "note-overview",
      "element_type": "note",
      "title": "Overview",
      "content": "Best-effort Bloomberg-compatible universal content scraper. Visits each URL in the batch URL list, scrolls for dynamically loaded content, extracts one structured row per page with url, title, cleaned JSON-wrapped content, author, published_at, format, and error_message. Bloomberg may show anti-bot, CAPTCHA, subscription, paywall, 403, or 404 pages; this template flags those conditions in error_message and leaves content empty for blocked/error pages. Pagination/navigation is implemented as a multi-URL batch loop using navigate.urls plus loop-continue; add valid article URLs to scrape all desired pages.",
      "color": "#f1c21b",
      "position_x": 80,
      "position_y": 20,
      "width": 480,
      "height": 160,
      "z_index": 22,
      "data": {}
    },
    {
      "id": "note-block-inject-javascript-1",
      "element_type": "note",
      "title": "Note: Inject JavaScript",
      "content": "Runs custom JavaScript in the page: `window.scrollTo(0, document.body.scrollHeight);...` Verify in browser if results are empty.",
      "color": "#ee5396",
      "position_x": 1664,
      "position_y": 200,
      "width": 340,
      "height": 122,
      "z_index": 22,
      "data": {
        "block_id": "inject-javascript-1"
      }
    },
    {
      "id": "note-block-structured-export-1",
      "element_type": "note",
      "title": "Note: Structured Export",
      "content": "Structured export with JS columns (url, title, content, author, published_at). These selectors are fragile — update if the site layout changes.",
      "color": "#ee5396",
      "position_x": 2336,
      "position_y": 200,
      "width": 340,
      "height": 127,
      "z_index": 22,
      "data": {
        "block_id": "structured-export-1"
      }
    },
    {
      "id": "note-block-loop-continue-1",
      "element_type": "note",
      "title": "Note: Loop Continue",
      "content": "Loop Continue advances a multi-URL or multi-text loop. Place at the end of the loop body with a clear back-edge to the loop start.",
      "color": "#ee5396",
      "position_x": 2672,
      "position_y": 200,
      "width": 340,
      "height": 123,
      "z_index": 22,
      "data": {
        "block_id": "loop-continue-1"
      }
    }
  ]
}