{ "version": "1.0.0", "exported_at": "2026-06-03T06:30:00.000Z", "project": { "name": "Google Shopping Listing Scraper for Australia", "description": "Extracts Google Shopping Australia listing data by keyword: keyword, product title, price, vendor, delivery, rating, review count, image URL, and product URL. The Octoparse sample encrypted-tbn URLs are Google Shopping image CDN URLs, not listing pages, so this template targets Google Shopping results on google.com.au using tbm=shop, hl=en-AU, gl=AU, pws=0, and num=100. Pagination is handled with a click-next loop and CSV append mode. Google may show consent, CAPTCHA, unusual-traffic blocks, or heavily variable markup; this template uses JavaScript normalization from rendered DOM plus raw HTML/script fallback parsing.", "color": "bg-[#4589ff]", "template_id": "ai-generated" }, "blocks": [ { "block_id": "navigate-1", "block_type": "process", "title": "Navigate", "description": "Go to a URL", "position_x": 100, "position_y": 240, "config": { "urls": [ "https://www.google.com.au/search?tbm=shop&hl=en-AU&gl=AU&pws=0&num=100&q=iPad%2013%20pro" ], "color": "bg-[#4589ff]", "tags": [ "google-shopping", "australia", "keyword-search" ] } }, { "block_id": "wait-for-page-load-1", "block_type": "process", "title": "Wait for Page Load", "description": "Wait for page to finish loading", "position_x": 460, "position_y": 240, "config": { "timeout": 30 } }, { "block_id": "inject-javascript-1", "block_type": "process", "title": "Inject JavaScript", "description": "Execute custom JavaScript", "position_x": 820, "position_y": 240, "config": { "jsCode": "(() => { const acceptRe = /(accept all|i agree|accept|agree|got it|continue)/i; const nodes = Array.from(document.querySelectorAll('button, div[role=\"button\"], input[type=\"submit\"], input[type=\"button\"], a')); const btn = nodes.find(el => acceptRe.test(((el.innerText || el.value || el.getAttribute('aria-label') || '') + '').trim())); if (btn) { btn.click(); return 'clicked-consent'; } return 'no-consent'; })();", "waitForCompletion": true, "timeout": 10 } }, { "block_id": "sleep-1", "block_type": "process", "title": "Sleep", "description": "Wait for specified time", "position_x": 1180, "position_y": 240, "config": { "duration": 6 } }, { "block_id": "wait-for-element-1", "block_type": "process", "title": "Wait for Element", "description": "Wait until element appears", "position_x": 1540, "position_y": 240, "config": { "selector": "body", "timeout": 30, "visible": true } }, { "block_id": "scroll-1", "block_type": "process", "title": "Scroll", "description": "Scroll page or element", "position_x": 1900, "position_y": 240, "config": { "direction": "down", "amount": 1400 } }, { "block_id": "sleep-2", "block_type": "process", "title": "Sleep", "description": "Wait for specified time", "position_x": 2260, "position_y": 240, "config": { "duration": 3 } }, { "block_id": "inject-javascript-2", "block_type": "process", "title": "Inject JavaScript", "description": "Execute custom JavaScript", "position_x": 2620, "position_y": 240, "config": { "jsCode": "(() => { document.querySelectorAll('#uscraper-generated-shopping-rows').forEach(e => e.remove()); const params = new URL(window.location.href).searchParams; const keyword = (params.get('q') || '').trim(); const priceRe = /(?:AU\\$|A\\$|AUD\\s?\\$?|\\$)\\s?\\d[\\d,.]*(?:\\.\\d{2})?/i; const priceReG = /(?:AU\\$|A\\$|AUD\\s?\\$?|\\$)\\s?\\d[\\d,.]*(?:\\.\\d{2})?/ig; const badPageRe = /unusual traffic|verify you are human|not a robot|captcha/i; const clean = s => ((s || '') + '').replace(/\\\\u0026/g, '&').replace(/\\\\x26/g, '&').replace(/\\\\u003d/g, '=').replace(/\\\\x3d/g, '=').replace(/\\\\\\//g, '/').replace(/&/g, '&').replace(/"/g, '\"').replace(/'/g, \"'\").replace(/\\s+/g, ' ').trim(); const bodyText = clean(document.body.innerText || document.body.textContent || ''); if (badPageRe.test(bodyText)) return 0; const container = document.createElement('div'); container.id = 'uscraper-generated-shopping-rows'; container.style.cssText = 'position:absolute;left:-10000px;top:0;width:1px;height:1px;overflow:hidden;'; document.body.appendChild(container); const seen = new Set(); const badLineRe = /^(shopping|images|all|maps|videos|news|books|flights|finance|search tools|feedback|cached|similar|about this result|sort by|filter|results|next|previous|google|sign in)$/i; const strip = s => clean(String(s || '').replace(//gi, ' ').replace(//gi, ' ').replace(/<[^>]+>/g, ' ').replace(/[{}\\[\\],]+/g, ' ')); const visible = el => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); const splitLines = txt => String(txt || '').split(/\\n|\\r| {2,}|\\t/).map(clean).filter(Boolean); const allImages = Array.from(document.querySelectorAll('img')).map(img => img.currentSrc || img.src || img.getAttribute('data-src') || '').filter(Boolean); function usefulLine(l) { if (!l) return false; if (l.length < 3 || l.length > 180) return false; if (badLineRe.test(l)) return false; if (priceRe.test(l)) return false; if (/https?:|encrypted-tbn|gstatic|googleusercontent|schema.org|function\\(|var |return |null|undefined/i.test(l)) return false; if (/delivery|shipping|pickup|reviews?|^\\([\\d,.]+\\s?[KkMm]?\\)$|rating|sponsored|compare prices|shop now|visit site/i.test(l)) return false; return true; } function firstUseful(lines, start, dir) { for (let n = 0; n < 12; n++) { const i = start + n * dir; if (usefulLine(lines[i])) return lines[i]; } return ''; } function emit(d) { d = d || {}; d.title = clean(d.title); d.price = clean(d.price); d.vendor = clean(d.vendor); d.delivery = clean(d.delivery); d.rating = clean(d.rating); d.review = clean(d.review); d.image = clean(d.image); d.url = clean(d.url); if (!d.price && !d.title) return false; if (!d.price || !priceRe.test(d.price)) return false; const key = [d.title, d.price, d.vendor, d.image].join('|').toLowerCase(); if (seen.has(key)) return false; seen.add(key); const row = document.createElement('div'); row.className = 'uscraper-shopping-row'; const fields = { keyword: d.keyword || keyword, title: d.title, price: d.price, vendor: d.vendor, delivery: d.delivery, rating: d.rating, review: d.review, image: d.image, url: d.url }; Object.entries(fields).forEach(([k, v]) => { const span = document.createElement('span'); span.className = 'us-' + k; span.textContent = clean(v); row.appendChild(span); }); container.appendChild(row); return true; } function parseElement(el, fallbackImage) { const raw = el ? ((el.innerText || el.textContent || '') + '') : ''; const text = clean(raw); const lines = splitLines(raw); const price = clean((text.match(priceRe) || [''])[0]); const priceIndex = lines.findIndex(l => priceRe.test(l)); const heading = el ? Array.from(el.querySelectorAll('h3, [role=\"heading\"], a, span')).map(x => clean(x.textContent)).find(usefulLine) || '' : ''; const title = heading || (priceIndex >= 0 ? firstUseful(lines, priceIndex - 1, -1) || firstUseful(lines, 0, 1) : firstUseful(lines, 0, 1)); const vendor = priceIndex >= 0 ? firstUseful(lines, priceIndex + 1, 1).replace(/^from\\s+/i, '') : ''; const delivery = lines.find(l => /(free\\s+delivery|delivery|shipping|pickup|arrives|by\\s+\\d{1,2}\\s+[A-Za-z]+)/i.test(l)) || ''; const ariaRating = el ? Array.from(el.querySelectorAll('[aria-label]')).map(x => x.getAttribute('aria-label') || '').find(t => /rated|rating|out of 5/i.test(t)) || '' : ''; const rating = clean(((ariaRating || text).match(/\\b[0-5](?:\\.\\d)?\\b(?=\\s*(?:out of 5|\\(?[\\d,.]+\\s?[KkMm]?\\)?|stars?))/i) || [''])[0]); const review = clean((text.match(/\\([\\d,.]+\\s?[KkMm]?\\)/) || text.match(/[\\d,.]+\\s?[KkMm]?\\s+reviews?/i) || [''])[0]); const img = el ? el.querySelector('img') : null; const image = clean((img && (img.currentSrc || img.src || img.getAttribute('data-src'))) || fallbackImage || ''); const link = el ? Array.from(el.querySelectorAll('a[href]')).map(a => a.href).find(h => h && !/^javascript:/i.test(h)) || '' : ''; return { keyword, title, price, vendor, delivery, rating, review, image, url: link }; } const candidates = []; function addCandidate(el) { if (!el || el === document.body || el === document.documentElement || candidates.includes(el)) return; const txt = clean(el.innerText || el.textContent || ''); if (!priceRe.test(txt)) return; if (txt.length < 8 || txt.length > 4000) return; const r = el.getBoundingClientRect ? el.getBoundingClientRect() : { width: 200, height: 100 }; if (visible(el) && (r.width < 80 || r.height < 40)) return; candidates.push(el); } document.querySelectorAll('div.sh-dgr__grid-result, div.sh-dgr__content, div.i0X6df, div[data-docid], div[data-merchant-id], g-inner-card, article, li, div[role=\"listitem\"], div[role=\"heading\"]').forEach(addCandidate); Array.from(document.querySelectorAll('span, div, a')).forEach(el => { const txt = clean(el.textContent || ''); if (!priceRe.test(txt)) return; let n = el; for (let depth = 0; n && depth < 10; depth++, n = n.parentElement) { const nt = clean(n.innerText || n.textContent || ''); if (priceRe.test(nt) && (n.querySelector('img') || nt.length < 1600)) { addCandidate(n); break; } } }); candidates.forEach(el => emit(parseElement(el, ''))); const raw0 = document.documentElement.innerHTML || ''; const raw = raw0.replace(/\\\\u0026/g, '&').replace(/\\\\x26/g, '&').replace(/\\\\u003d/g, '=').replace(/\\\\x3d/g, '=').replace(/\\\\u0022/g, '\"').replace(/\\\\x22/g, '\"').replace(/\\\\\\//g, '/'); const imageRe = /https?:\\/\\/encrypted-tbn\\d?\\.gstatic\\.com\\/shopping\\?q=tbn:[^\"'<>\\\\\\s]+/ig; const rawImages = Array.from(raw.matchAll(imageRe)).map(m => clean(m[0])).filter(Boolean); const matches = Array.from(raw.matchAll(priceReG)); matches.slice(0, 160).forEach((m, idx) => { const pos = m.index || 0; const chunk = raw.slice(Math.max(0, pos - 2500), Math.min(raw.length, pos + 2500)); const text = strip(chunk).replace(/[\"'|]/g, '\\n'); const lines = splitLines(text); const price = clean(m[0]); const pidx = lines.findIndex(l => l.includes(price) || priceRe.test(l)); const title = (pidx >= 0 ? firstUseful(lines, pidx - 1, -1) || firstUseful(lines, pidx + 1, 1) : firstUseful(lines, 0, 1)); const vendor = pidx >= 0 ? firstUseful(lines, pidx + 1, 1).replace(/^from\\s+/i, '') : ''; const delivery = lines.find(l => /(free\\s+delivery|delivery|shipping|pickup|arrives|by\\s+\\d{1,2}\\s+[A-Za-z]+)/i.test(l)) || ''; const review = clean((text.match(/\\([\\d,.]+\\s?[KkMm]?\\)/) || text.match(/[\\d,.]+\\s?[KkMm]?\\s+reviews?/i) || [''])[0]); const rating = clean((text.match(/\\b[0-5](?:\\.\\d)?\\b(?=\\s*(?:out of 5|stars?|\\(?[\\d,.]+\\s?[KkMm]?\\)?))/i) || [''])[0]); const img = clean((chunk.match(imageRe) || [])[0] || rawImages[idx] || allImages[idx] || ''); const urlMatch = chunk.match(/https?:\\/\\/(?:www\\.)?google\\.com(?:\\.au)?\\/shopping\\/product[^\"'<>\\\\\\s]+|\\/shopping\\/product[^\"'<>\\\\\\s]+/i); const url = urlMatch ? (urlMatch[0].startsWith('http') ? urlMatch[0] : 'https://www.google.com.au' + urlMatch[0]) : ''; emit({ keyword, title, price, vendor, delivery, rating, review, image: img, url }); }); if (container.children.length === 0) { const lines = (document.body.innerText || '').split('\\n').map(clean).filter(Boolean); const priceIndexes = lines.map((l, i) => priceRe.test(l) ? i : -1).filter(i => i >= 0); priceIndexes.slice(0, 100).forEach((idx, n) => { const title = firstUseful(lines, idx - 1, -1) || firstUseful(lines, idx + 1, 1); const price = clean((lines[idx].match(priceRe) || [''])[0]); const vendor = firstUseful(lines, idx + 1, 1); const delivery = lines.slice(idx, idx + 8).find(l => /(free\\s+delivery|delivery|shipping|pickup|arrives)/i.test(l)) || ''; emit({ keyword, title, price, vendor, delivery, rating: '', review: '', image: allImages[n] || rawImages[n] || '', url: '' }); }); } return container.children.length; })();", "waitForCompletion": true, "timeout": 20 } }, { "block_id": "element-exists-1", "block_type": "process", "title": "Element Exists", "description": "Check if element exists", "position_x": 2980, "position_y": 240, "config": { "selector": ".uscraper-shopping-row" } }, { "block_id": "structured-export-1", "block_type": "process", "title": "Structured Export", "description": "Export data with custom columns", "position_x": 3340, "position_y": 240, "config": { "rowSelector": ".uscraper-shopping-row", "fileName": "google-shopping-listing-scraper-for-au.csv", "saveLocation": "C:\\Users\\theskd\\Documents\\UScraper\\templates", "includeHeaders": true, "fileMode": "append", "columns": [ { "name": "keyword", "selector": ".us-keyword", "attribute": "text" }, { "name": "product_title", "selector": ".us-title", "attribute": "text" }, { "name": "price", "selector": ".us-price", "attribute": "text" }, { "name": "vendor", "selector": ".us-vendor", "attribute": "text" }, { "name": "delivery", "selector": ".us-delivery", "attribute": "text" }, { "name": "rating", "selector": ".us-rating", "attribute": "text" }, { "name": "review", "selector": ".us-review", "attribute": "text" }, { "name": "image", "selector": ".us-image", "attribute": "text" }, { "name": "product_url", "selector": ".us-url", "attribute": "text" } ] } }, { "block_id": "element-exists-2", "block_type": "process", "title": "Element Exists", "description": "Check if element exists", "position_x": 3700, "position_y": 240, "config": { "selector": "a#pnnext, a[aria-label=\"Next page\"], a[aria-label=\"Next\"], a[aria-label*=\"Next\"]" } }, { "block_id": "click-1", "block_type": "process", "title": "Click", "description": "Click on element", "position_x": 4060, "position_y": 560, "config": { "selector": "a#pnnext, a[aria-label=\"Next page\"], a[aria-label=\"Next\"], a[aria-label*=\"Next\"]", "timeout": 10 } }, { "block_id": "wait-for-page-load-2", "block_type": "process", "title": "Wait for Page Load", "description": "Wait for page to finish loading", "position_x": 4420, "position_y": 560, "config": { "timeout": 30 } }, { "block_id": "sleep-3", "block_type": "process", "title": "Sleep", "description": "Wait for specified time", "position_x": 4780, "position_y": 560, "config": { "duration": 4 } }, { "block_id": "loop-continue-1", "block_type": "process", "title": "Loop Continue", "description": "Continue multi-input loop", "position_x": 3700, "position_y": 880, "config": {} } ], "connections": [ { "from_block_id": "navigate-1", "from_connector_id": "right", "to_block_id": "wait-for-page-load-1", "to_connector_id": "left" }, { "from_block_id": "wait-for-page-load-1", "from_connector_id": "right", "to_block_id": "inject-javascript-1", "to_connector_id": "left" }, { "from_block_id": "inject-javascript-1", "from_connector_id": "right", "to_block_id": "sleep-1", "to_connector_id": "left" }, { "from_block_id": "sleep-1", "from_connector_id": "right", "to_block_id": "wait-for-element-1", "to_connector_id": "left" }, { "from_block_id": "wait-for-element-1", "from_connector_id": "right", "to_block_id": "scroll-1", "to_connector_id": "left" }, { "from_block_id": "scroll-1", "from_connector_id": "right", "to_block_id": "sleep-2", "to_connector_id": "left" }, { "from_block_id": "sleep-2", "from_connector_id": "right", "to_block_id": "inject-javascript-2", "to_connector_id": "left" }, { "from_block_id": "inject-javascript-2", "from_connector_id": "right", "to_block_id": "element-exists-1", "to_connector_id": "left" }, { "from_block_id": "element-exists-1", "from_connector_id": "true", "to_block_id": "structured-export-1", "to_connector_id": "left" }, { "from_block_id": "element-exists-1", "from_connector_id": "false", "to_block_id": "loop-continue-1", "to_connector_id": "left" }, { "from_block_id": "structured-export-1", "from_connector_id": "right", "to_block_id": "element-exists-2", "to_connector_id": "left" }, { "from_block_id": "element-exists-2", "from_connector_id": "true", "to_block_id": "click-1", "to_connector_id": "left" }, { "from_block_id": "element-exists-2", "from_connector_id": "false", "to_block_id": "loop-continue-1", "to_connector_id": "left" }, { "from_block_id": "click-1", "from_connector_id": "right", "to_block_id": "wait-for-page-load-2", "to_connector_id": "left" }, { "from_block_id": "wait-for-page-load-2", "from_connector_id": "right", "to_block_id": "sleep-3", "to_connector_id": "left" }, { "from_block_id": "sleep-3", "from_connector_id": "right", "to_block_id": "inject-javascript-2", "to_connector_id": "left" } ], "canvas_elements": [ { "id": "group-load", "element_type": "group", "title": "Page Load", "color": "#08bdba", "position_x": 28, "position_y": 136, "width": 5000, "height": 616, "z_index": 20, "data": { "memberBlockIds": [ "navigate-1", "wait-for-page-load-1", "sleep-1", "wait-for-element-1", "sleep-2", "wait-for-page-load-2", "sleep-3" ] } }, { "id": "group-interaction", "element_type": "group", "title": "Interaction", "color": "#a56eff", "position_x": 748, "position_y": 136, "width": 2120, "height": 296, "z_index": 20, "data": { "memberBlockIds": [ "inject-javascript-1", "scroll-1", "inject-javascript-2" ] } }, { "id": "group-pagination", "element_type": "group", "title": "Pagination Loop", "color": "#ff832b", "position_x": 2908, "position_y": 136, "width": 1400, "height": 936, "z_index": 20, "data": { "memberBlockIds": [ "element-exists-1", "element-exists-2", "click-1", "loop-continue-1" ] } }, { "id": "group-extract", "element_type": "group", "title": "Data Extraction", "color": "#42be65", "position_x": 3268, "position_y": 136, "width": 380, "height": 296, "z_index": 20, "data": { "memberBlockIds": [ "structured-export-1" ] } }, { "id": "note-overview", "element_type": "note", "title": "Overview", "content": "Extracts Google Shopping Australia listing data by keyword: keyword, product title, price, vendor, delivery, rating, review count, image URL, and product URL. The Octoparse sample encrypted-tbn URLs are Google Shopping image CDN URLs, not listing pages, so this template targets Google Shopping results on google.com.au using tbm=shop, hl=en-AU, gl=AU, pws=0, and num=100. Pagination is handled with a click-next loop and CSV append mode. Google may show consent, CAPTCHA, unusual-traffic blocks, or heavily variable markup; this template uses JavaScript normalization from rendered DOM plus raw HTML/script fallback parsing.", "color": "#f1c21b", "position_x": 80, "position_y": 20, "width": 480, "height": 160, "z_index": 22, "data": {} }, { "id": "note-block-inject-javascript-1", "element_type": "note", "title": "Note: Inject JavaScript", "content": "Runs custom JavaScript in the page: `(() => { const acceptRe = /(accept all|i agree|accept|agree|got it|continue)/i; const nodes = Array....` Verify in browser if results are empty.", "color": "#ee5396", "position_x": 1020, "position_y": 220, "width": 340, "height": 140, "z_index": 22, "data": { "block_id": "inject-javascript-1" } }, { "id": "note-block-inject-javascript-2", "element_type": "note", "title": "Note: Inject JavaScript", "content": "Runs custom JavaScript in the page: `(() => { document.querySelectorAll('#uscraper-generated-shopping-rows').forEach(e => e.remove()); co...` Verify in browser if results are empty.", "color": "#ee5396", "position_x": 2820, "position_y": 220, "width": 340, "height": 140, "z_index": 22, "data": { "block_id": "inject-javascript-2" } }, { "id": "note-block-element-exists-1", "element_type": "note", "title": "Note: Element Exists", "content": "Condition block: checks `.uscraper-shopping-row`. True / False branches control which path runs next. Keep enough space between branches so both connector lines are visible.", "color": "#ee5396", "position_x": 3180, "position_y": 220, "width": 340, "height": 137, "z_index": 22, "data": { "block_id": "element-exists-1" } }, { "id": "note-block-structured-export-1", "element_type": "note", "title": "Note: Structured Export", "content": "Extracts rows matching `.uscraper-shopping-row`. Confirm row count > 0 before running at scale.", "color": "#ee5396", "position_x": 3540, "position_y": 220, "width": 340, "height": 111, "z_index": 22, "data": { "block_id": "structured-export-1" } }, { "id": "note-block-element-exists-2", "element_type": "note", "title": "Note: Element Exists", "content": "Condition block: checks `a#pnnext, a[aria-label=\"Next page\"], a[aria-label=\"Next\"], a[aria-label*=\"Next\"]`. True / False branches control which path runs next. Keep enough space between branches so both connector lines are visible.", "color": "#ee5396", "position_x": 3900, "position_y": 220, "width": 340, "height": 157, "z_index": 22, "data": { "block_id": "element-exists-2" } }, { "id": "note-block-click-1", "element_type": "note", "title": "Note: Click", "content": "Pagination click — add waits after this block; the page reloads asynchronously.", "color": "#ee5396", "position_x": 4260, "position_y": 540, "width": 316, "height": 106, "z_index": 22, "data": { "block_id": "click-1" } }, { "id": "note-block-loop-continue-1", "element_type": "note", "title": "Note: Loop Continue", "content": "Loop Continue advances a multi-URL or multi-text loop. Place at the end of the loop body with a clear back-edge to the loop start.", "color": "#ee5396", "position_x": 3900, "position_y": 860, "width": 340, "height": 123, "z_index": 22, "data": { "block_id": "loop-continue-1" } } ] }