const axios = require("axios");
// API URL
const API_URL = "https://openapi.coreclaw.com/api/v1/scraper/run";
// Your API KEY
const API_KEY = "<YOUR_API_KEY>";
// Callback URL, The endpoint that will receive the scraping results
const CALLBACK_URL = "https://your-domain.com/callback";
// Build request parameters
const requestBody = {
"scraper_slug": "01KG2DV66JTCN65ZBTRX3M456E",
"version": "v1.0.8",
"input": {
"parameters": {
"system": {
"proxy_region": "",
"cpus": 0.125,
"memory": 512,
"execute_limit_time_seconds": 1800,
"max_total_charge": 0,
"max_total_traffic": 0,
},
"custom": {
"collection_type": "hashtag",
"targets": [
{
"string": "fyp"
}
],
"fetch_count": 10,
"fetch_comments": false,
"comment_max_per_post": null,
"comment_top_level_max_per_post": 0,
"comment_reply_max_per_comment": 0,
"comment_max_pages": 10,
"video_fetch_count": 10,
"profile_scrape_section": "Videos",
"profile_video_sort": "Latest",
"exclude_pinned_posts": false,
"filter_publish_after_date": "",
"filter_publish_before_date": "",
"filter_heart_less_than": null,
"filter_heart_gte_than": null,
"max_follower_fetch_num": 0,
"max_following_fetch_num": 0,
"scrape_related_videos": false,
"retry_times": 3,
"retry_delay_seconds": 1
}
}
},
"callback_url": CALLBACK_URL
};
async function runScraper() {
try {
const response = await axios.post(API_URL, requestBody, {
headers: {
"api-key": API_KEY,
"Content-Type": "application/json"
},
timeout: 30000
});
const { code, message, data } = response.data
// Check response status
if(code === 0) {
console.log("Status Code:", code)
console.log("Response Message:", message)
console.log("Response Body:", JSON.stringify(data))
} else {
console.log("Error Status Code:", code)
console.log("Error Message:", message)
}
} catch (error) {
console.error("Request failed:", error.message);
}
}
// Execute
runScraper();