Shopify 产品抓取工具

{
    "startUrl": [{ "url": "https://www.gymshark.com" }],
    "maxRequestsPerCrawl": 0,
    "maxConcurrency": 20,
    "checkForBanner": true,
    "fetchHtml": false,
    "debugLog": false,
    "extendOutputFunction": "async ({ data, item, product, images, fns, name, request, variants, context, customData, input, platform }) => {
  return item;
}",
    "extendScraperFunction": "async ({ fns, customData, platform, label }) => {
 
}",
    "customData": {}
}

{
    "startUrl": [{ "url": "https://www.gymshark.com" }],
    "maxRequestsPerCrawl": 0,
    "maxConcurrency": 20
}

{
    "startUrl": [
        { "url": "https://www.gymshark.com" },
        { "url": "https://www.spanx.com" },
        { "url": "https://www.nativecos.com" }
    ],
    "maxRequestsPerCrawl": 0,
    "maxConcurrency": 20
}

async ({ data, item, fns, input, platform }) => {
    // 过滤掉低价产品
    if (item.price < 10) return null;

    // 计算折扣百分比
    const comparePrice = data.product?.variants?.[0]?.compare_at_price;
    if (comparePrice && item.price) {
        item.additional.discount_pct = Math.round((1 - item.price / comparePrice) * 100);
    }

    // 排除礼品卡类产品
    if (item.product_type === 'Gift Card') return null;

    return item;
}

async ({ fns, customData, platform, label }) => {
    if (label === 'SETUP') {
        // SETUP 阶段可访问 requestQueue 注入额外 URL
        const extraSitemap = customData.extraSitemapUrl;
        if (extraSitemap) {
            // 通过传入的 { requestQueue } 添加请求
        }
    }

    if (label === 'FILTER_SITEMAP_URL') {
        // this.url 是正在评估的产品/sitemap URL
        // this.filter(false) 可将其排除
    }

    if (label === 'PRENAVIGATION') {
        // this.crawlingContext.request — 修改请求头
    }

    if (label === 'FINISHED') {
        // this.crawler — 访问爬虫统计、持久化状态
    }
}

{
    "startUrl": [{ "url": "https://www.colourpop.com" }],
    "maxRequestsPerCrawl": 20,
    "fetchHtml": true
}

async ({ data, item, request, context }) => {
    const htmlBody = request.userData.body;
    if (htmlBody) {
        // 从 HTML 中解析 JSON-LD 或 meta 标签
        // context.$ 是从 HTML 页面加载的 cheerio 实例
    }
    return item;
}

{
    "startUrl": [{ "url": "https://www.kith.com" }],
    "maxRequestsPerCrawl": 5,
    "debugLog": true
}