数据集合并和去重工具

const axios = require("axios");

// API URL
const API_URL = "https://openapi.coreclaw.com/api/v1/scraper/run";

// Your API KEY
const API_KEY = "<YOUR_API_KEY>";

// Callback URL, The endpoint that will receive the scraping results
const CALLBACK_URL = "https://your-domain.com/callback";

// Build request parameters
const requestBody = {
    "scraper_slug": "01KG2DV66JTCN65ZBTRX3M456E",
    "version": "v1.0.8",
    "input": {
        "parameters": {
            "system": {
                "proxy_region": "",
                "cpus": 0.125,
                "memory": 512,
                "execute_limit_time_seconds": 1800,
                "max_total_charge": 0,
                "max_total_traffic": 0,
            },
            "custom": {
          "runUnits": [
                    {
                              "url": "https://coreclaw.local/__single_run__"
                    }
          ],
          "scenario": "ecommerce-products",
          "fields": [
                    {
                              "string": "productId"
                    },
                    {
                              "string": "sku"
                    }
          ],
          "mergeStrategy": "keep-newest",
          "timestampField": "updatedAt",
          "dataSourceType": "direct-input",
          "inputData": "[{\"productId\": \"P001\", \"sku\": \"SKU-A-BLACK\", \"name\": \"无线蓝牙耳机 Pro\", \"price\": 299.00, \"stock\": 156, \"source\": \"京东旗舰店\", \"updatedAt\": \"2024-01-20T10:30:00\"}, {\"productId\": \"P001\", \"sku\": \"SKU-A-BLACK\", \"name\": \"无线蓝牙耳机 Pro (黑)\", \"price\": 279.00, \"stock\": 200, \"source\": \"天猫旗舰店\", \"updatedAt\": \"2024-01-22T14:20:00\"}, {\"productId\": \"P001\", \"sku\": \"SKU-A-WHITE\", \"name\": \"无线蓝牙耳机 Pro\", \"price\": 299.00, \"stock\": 88, \"source\": \"京东旗舰店\", \"updatedAt\": \"2024-01-20T10:30:00\"}, {\"productId\": \"P002\", \"sku\": \"SKU-B\", \"name\": \"智能手表 Ultra\", \"price\": 1299.00, \"stock\": 45, \"source\": \"官网\", \"updatedAt\": \"2024-01-18T09:00:00\"}]",
          "inputUrls": [
                    {
                              "url": "https://raw.githubusercontent.com/kael-odin/worker-dedup-datasets/main/test/data1.json"
                    }
          ],
          "datasetIds": [],
          "inputFormat": "json",
          "output": "unique-items",
          "generateReport": true,
          "mode": "dedup-after-load",
          "fieldsToLoad": [],
          "nullAsUnique": false,
          "parallelLoads": 10,
          "parallelPushes": 5,
          "batchSize": 5000,
          "appendFileSource": false,
          "verboseLog": false
}
        }
    },
    "callback_url": CALLBACK_URL
};

async function runScraper() {
    try {
        const response = await axios.post(API_URL, requestBody, {
            headers: {
                "api-key": API_KEY,
                "Content-Type": "application/json"
            },
            timeout: 30000
        });

        const { code, message, data } = response.data

        // Check response status
        if(code === 0) {
            console.log("Status Code:", code)
            console.log("Response Message:", message)
            console.log("Response Body:", JSON.stringify(data))
        } else {
            console.log("Error Status Code:", code)
            console.log("Error Message:", message)
        }
    } catch (error) {
        console.error("Request failed:", error.message);
    }
}

// Execute
runScraper();