forked from zhouruizhe/gmTouringMiniApp
1040 lines
28 KiB
JavaScript
1040 lines
28 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { dirname, relative, resolve, sep } from 'node:path'
|
|
import process from 'node:process'
|
|
import { pathToFileURL } from 'node:url'
|
|
|
|
export const AMAP_ENDPOINT = 'https://restapi.amap.com/v5/place/text'
|
|
export const TARGET_ADCODE = '440311'
|
|
export const MAX_RESULTS_PER_QUERY = 200
|
|
|
|
export const DEFAULT_TYPE_GROUPS = Object.freeze([
|
|
Object.freeze({
|
|
name: 'scenic-attractions',
|
|
types: Object.freeze(['110000']),
|
|
}),
|
|
Object.freeze({
|
|
name: 'public-culture',
|
|
types: Object.freeze([
|
|
'140100',
|
|
'140200',
|
|
'140300',
|
|
'140400',
|
|
'140500',
|
|
'140600',
|
|
'140700',
|
|
'140800',
|
|
]),
|
|
}),
|
|
Object.freeze({
|
|
name: 'leisure-and-performing-arts',
|
|
types: Object.freeze([
|
|
'080401',
|
|
'080501',
|
|
'080502',
|
|
'080503',
|
|
'080504',
|
|
'080505',
|
|
'080602',
|
|
'080603',
|
|
]),
|
|
}),
|
|
Object.freeze({
|
|
name: 'farms',
|
|
types: Object.freeze(['170402']),
|
|
}),
|
|
])
|
|
|
|
export const DEFAULT_KEYWORDS = Object.freeze([
|
|
'景区',
|
|
'公园',
|
|
'森林公园',
|
|
'湿地公园',
|
|
'郊野公园',
|
|
'绿道',
|
|
'博物馆',
|
|
'纪念馆',
|
|
'文化馆',
|
|
'艺术馆',
|
|
'美术馆',
|
|
'图书馆',
|
|
'展览馆',
|
|
'科技馆',
|
|
'剧院',
|
|
'农场',
|
|
'田园',
|
|
'生态园',
|
|
'采摘园',
|
|
'古村',
|
|
'非遗',
|
|
])
|
|
|
|
const RETRYABLE_AMAP_INFOCODES = new Set([
|
|
'10004',
|
|
'10016',
|
|
'10019',
|
|
'10020',
|
|
'10021',
|
|
])
|
|
|
|
const PROTECTED_OUTPUT_DIRECTORIES = ['src', 'scripts', 'test', 'tests']
|
|
|
|
const HELP_TEXT = `
|
|
Collect unreviewed AMap POI 2.0 candidates for Shenzhen Guangming District.
|
|
|
|
Usage:
|
|
AMAP_WEB_SERVICE_KEY=... node scripts/fetch-amap-pois.mjs [options]
|
|
|
|
Options:
|
|
--output <file> JSON output path. Defaults to the system temp dir.
|
|
--type-group <name=codes> Repeatable group; codes are separated with "|".
|
|
Custom groups replace the default type groups.
|
|
--keyword <text> Repeatable supplementary keyword.
|
|
Custom keywords replace the default keywords.
|
|
--no-types Disable default type-group queries.
|
|
--no-keywords Disable default keyword queries.
|
|
--page-size <1..25> Results requested per page. Default: 25.
|
|
--max-pages <number> Pages per query. Default: 8; never over 200 results.
|
|
--throttle-ms <number> Minimum delay between requests. Default: 500.
|
|
--retries <0..10> Retries for transient failures. Default: 3.
|
|
--retry-base-ms <number> Exponential retry base delay. Default: 1000.
|
|
--timeout-ms <number> Per-request timeout. Default: 15000.
|
|
--overwrite Replace an existing output file.
|
|
--dry-run Print the query plan without reading the API key.
|
|
--help Show this help.
|
|
|
|
The region is fixed to adcode 440311 with city_limit=true. The API key can only
|
|
be read from AMAP_WEB_SERVICE_KEY and is never written to output or logs.
|
|
Candidate data must be reviewed before it is published.
|
|
`
|
|
|
|
class PublicError extends Error {
|
|
constructor(message, { code = 'ERROR', retryable = false } = {}) {
|
|
super(message)
|
|
this.name = 'PublicError'
|
|
this.code = code
|
|
this.retryable = retryable
|
|
}
|
|
}
|
|
|
|
function isRecord(value) {
|
|
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
}
|
|
|
|
function toText(value) {
|
|
if (Array.isArray(value)) {
|
|
for (const entry of value) {
|
|
const text = toText(entry)
|
|
if (text !== null)
|
|
return text
|
|
}
|
|
return null
|
|
}
|
|
|
|
if (typeof value === 'string') {
|
|
const text = value.trim()
|
|
return text === '' ? null : text
|
|
}
|
|
|
|
if (typeof value === 'number' && Number.isFinite(value))
|
|
return String(value)
|
|
|
|
return null
|
|
}
|
|
|
|
function toFiniteNumber(value) {
|
|
const text = toText(value)
|
|
if (text === null)
|
|
return null
|
|
|
|
const number = Number(text)
|
|
return Number.isFinite(number) ? number : null
|
|
}
|
|
|
|
export function parseAmapLocation(value) {
|
|
const text = toText(value)
|
|
if (text === null)
|
|
return null
|
|
|
|
const [longitudeText, latitudeText, ...rest] = text.split(',')
|
|
if (rest.length > 0)
|
|
return null
|
|
|
|
const longitude = Number(longitudeText)
|
|
const latitude = Number(latitudeText)
|
|
if (
|
|
!Number.isFinite(longitude)
|
|
|| !Number.isFinite(latitude)
|
|
|| longitude < -180
|
|
|| longitude > 180
|
|
|| latitude < -90
|
|
|| latitude > 90
|
|
) {
|
|
return null
|
|
}
|
|
|
|
return { longitude, latitude }
|
|
}
|
|
|
|
function normalizePhoto(photo) {
|
|
if (!isRecord(photo))
|
|
return null
|
|
|
|
const url = toText(photo.url)
|
|
if (url === null)
|
|
return null
|
|
|
|
return {
|
|
title: toText(photo.title),
|
|
url,
|
|
}
|
|
}
|
|
|
|
function normalizePhotos(value) {
|
|
const entries = Array.isArray(value) ? value : isRecord(value) ? [value] : []
|
|
const photos = []
|
|
const seenUrls = new Set()
|
|
|
|
for (const entry of entries) {
|
|
const photo = normalizePhoto(entry)
|
|
if (photo === null || seenUrls.has(photo.url))
|
|
continue
|
|
seenUrls.add(photo.url)
|
|
photos.push(photo)
|
|
}
|
|
|
|
return photos
|
|
}
|
|
|
|
function normalizeQueryMatch(query) {
|
|
if (!isRecord(query))
|
|
return null
|
|
|
|
const id = toText(query.id)
|
|
const kind = toText(query.kind)
|
|
const label = toText(query.label)
|
|
if (id === null || kind === null || label === null)
|
|
return null
|
|
|
|
return { id, kind, label }
|
|
}
|
|
|
|
export function isPoiInAdcode(rawPoi, targetAdcode = TARGET_ADCODE) {
|
|
return isRecord(rawPoi) && toText(rawPoi.adcode) === targetAdcode
|
|
}
|
|
|
|
export function normalizeAmapPoi(rawPoi, query = null) {
|
|
if (!isRecord(rawPoi))
|
|
return null
|
|
|
|
const id = toText(rawPoi.id)
|
|
const name = toText(rawPoi.name)
|
|
const location = parseAmapLocation(rawPoi.location)
|
|
if (id === null || name === null || location === null)
|
|
return null
|
|
|
|
const business = isRecord(rawPoi.business) ? rawPoi.business : {}
|
|
const match = normalizeQueryMatch(query)
|
|
|
|
return {
|
|
id,
|
|
source: 'amap',
|
|
name,
|
|
parentId: toText(rawPoi.parent),
|
|
longitude: location.longitude,
|
|
latitude: location.latitude,
|
|
coordinateSystem: 'GCJ02',
|
|
type: toText(rawPoi.type),
|
|
typecode: toText(rawPoi.typecode),
|
|
provinceName: toText(rawPoi.pname),
|
|
provinceCode: toText(rawPoi.pcode),
|
|
cityName: toText(rawPoi.cityname),
|
|
cityCode: toText(rawPoi.citycode),
|
|
districtName: toText(rawPoi.adname),
|
|
adcode: toText(rawPoi.adcode),
|
|
address: toText(rawPoi.address),
|
|
business: {
|
|
alias: toText(business.alias),
|
|
telephone: toText(business.tel),
|
|
sourceRating: toFiniteNumber(business.rating),
|
|
openingHoursToday: toText(business.opentime_today),
|
|
openingHoursWeek: toText(business.opentime_week),
|
|
},
|
|
photos: normalizePhotos(rawPoi.photos),
|
|
matchedQueries: match === null ? [] : [match],
|
|
sourceOccurrences: 1,
|
|
}
|
|
}
|
|
|
|
export function normalizeAmapPage(rawPois, query, targetAdcode = TARGET_ADCODE) {
|
|
const result = {
|
|
pois: [],
|
|
filteredOutByAdcode: 0,
|
|
rejectedInvalid: 0,
|
|
}
|
|
|
|
if (!Array.isArray(rawPois))
|
|
return result
|
|
|
|
for (const rawPoi of rawPois) {
|
|
if (!isPoiInAdcode(rawPoi, targetAdcode)) {
|
|
result.filteredOutByAdcode += 1
|
|
continue
|
|
}
|
|
|
|
const poi = normalizeAmapPoi(rawPoi, query)
|
|
if (poi === null) {
|
|
result.rejectedInvalid += 1
|
|
continue
|
|
}
|
|
|
|
result.pois.push(poi)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
function firstPresent(first, second) {
|
|
if (first !== null && first !== undefined && first !== '')
|
|
return first
|
|
return second ?? null
|
|
}
|
|
|
|
function mergePhotos(first, second) {
|
|
const merged = []
|
|
const byUrl = new Map()
|
|
|
|
for (const photo of [...first, ...second]) {
|
|
if (!isRecord(photo))
|
|
continue
|
|
const url = toText(photo.url)
|
|
if (url === null)
|
|
continue
|
|
|
|
const existing = byUrl.get(url)
|
|
if (existing) {
|
|
existing.title = firstPresent(existing.title, toText(photo.title))
|
|
continue
|
|
}
|
|
|
|
const normalized = { title: toText(photo.title), url }
|
|
byUrl.set(url, normalized)
|
|
merged.push(normalized)
|
|
}
|
|
|
|
return merged
|
|
}
|
|
|
|
function mergeQueryMatches(first, second) {
|
|
const matches = []
|
|
const seenIds = new Set()
|
|
|
|
for (const entry of [...first, ...second]) {
|
|
const match = normalizeQueryMatch(entry)
|
|
if (match === null || seenIds.has(match.id))
|
|
continue
|
|
seenIds.add(match.id)
|
|
matches.push(match)
|
|
}
|
|
|
|
return matches
|
|
}
|
|
|
|
function cloneCandidate(poi) {
|
|
return {
|
|
...poi,
|
|
business: { ...(isRecord(poi.business) ? poi.business : {}) },
|
|
photos: mergePhotos([], Array.isArray(poi.photos) ? poi.photos : []),
|
|
matchedQueries: mergeQueryMatches(
|
|
[],
|
|
Array.isArray(poi.matchedQueries) ? poi.matchedQueries : [],
|
|
),
|
|
sourceOccurrences: Number.isInteger(poi.sourceOccurrences)
|
|
? poi.sourceOccurrences
|
|
: 1,
|
|
}
|
|
}
|
|
|
|
function mergeCandidates(first, second) {
|
|
const merged = cloneCandidate(first)
|
|
const next = cloneCandidate(second)
|
|
const scalarFields = [
|
|
'name',
|
|
'parentId',
|
|
'type',
|
|
'typecode',
|
|
'provinceName',
|
|
'provinceCode',
|
|
'cityName',
|
|
'cityCode',
|
|
'districtName',
|
|
'adcode',
|
|
'address',
|
|
]
|
|
|
|
for (const field of scalarFields)
|
|
merged[field] = firstPresent(merged[field], next[field])
|
|
|
|
const businessFields = [
|
|
'alias',
|
|
'telephone',
|
|
'sourceRating',
|
|
'openingHoursToday',
|
|
'openingHoursWeek',
|
|
]
|
|
for (const field of businessFields) {
|
|
merged.business[field] = firstPresent(
|
|
merged.business[field],
|
|
next.business[field],
|
|
)
|
|
}
|
|
|
|
merged.photos = mergePhotos(merged.photos, next.photos)
|
|
merged.matchedQueries = mergeQueryMatches(
|
|
merged.matchedQueries,
|
|
next.matchedQueries,
|
|
)
|
|
merged.sourceOccurrences += next.sourceOccurrences
|
|
return merged
|
|
}
|
|
|
|
export function dedupePois(pois) {
|
|
if (!Array.isArray(pois))
|
|
return []
|
|
|
|
const byId = new Map()
|
|
for (const poi of pois) {
|
|
if (!isRecord(poi))
|
|
continue
|
|
const id = toText(poi.id)
|
|
if (id === null)
|
|
continue
|
|
|
|
const normalized = cloneCandidate({ ...poi, id })
|
|
const existing = byId.get(id)
|
|
byId.set(id, existing ? mergeCandidates(existing, normalized) : normalized)
|
|
}
|
|
|
|
return [...byId.values()]
|
|
}
|
|
|
|
function validateTypeGroup(value) {
|
|
const separatorIndex = value.indexOf('=')
|
|
if (separatorIndex <= 0 || separatorIndex === value.length - 1) {
|
|
throw new PublicError(
|
|
'Invalid --type-group. Expected a name and six-digit codes, for example scenic=110000.',
|
|
{ code: 'INVALID_ARGUMENT' },
|
|
)
|
|
}
|
|
|
|
const name = value.slice(0, separatorIndex).trim()
|
|
const types = value
|
|
.slice(separatorIndex + 1)
|
|
.split('|')
|
|
.map(type => type.trim())
|
|
.filter(Boolean)
|
|
|
|
if (!/^[\w-]+$/.test(name) || types.length === 0) {
|
|
throw new PublicError('Invalid --type-group.', { code: 'INVALID_ARGUMENT' })
|
|
}
|
|
if (types.some(type => !/^\d{6}$/.test(type))) {
|
|
throw new PublicError('Every POI type code must contain exactly six digits.', {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
return { name, types: [...new Set(types)] }
|
|
}
|
|
|
|
function readOptionValue(argv, index, inlineValue, optionName) {
|
|
if (inlineValue !== null)
|
|
return { value: inlineValue, nextIndex: index }
|
|
|
|
const value = argv[index + 1]
|
|
if (value === undefined || value.startsWith('--')) {
|
|
throw new PublicError(`Missing value for ${optionName}.`, {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
return { value, nextIndex: index + 1 }
|
|
}
|
|
|
|
function parseInteger(value, name, { minimum, maximum = Number.MAX_SAFE_INTEGER }) {
|
|
if (!/^\d+$/.test(value)) {
|
|
throw new PublicError(`${name} must be an integer.`, {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
const number = Number(value)
|
|
if (!Number.isSafeInteger(number) || number < minimum || number > maximum) {
|
|
throw new PublicError(`${name} must be between ${minimum} and ${maximum}.`, {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
return number
|
|
}
|
|
|
|
export function parseCliArgs(argv) {
|
|
const config = {
|
|
output: null,
|
|
pageSize: 25,
|
|
maxPages: null,
|
|
throttleMs: 500,
|
|
retries: 3,
|
|
retryBaseMs: 1000,
|
|
timeoutMs: 15000,
|
|
overwrite: false,
|
|
dryRun: false,
|
|
help: false,
|
|
defaultTypesEnabled: true,
|
|
defaultKeywordsEnabled: true,
|
|
customTypeGroups: [],
|
|
customKeywords: [],
|
|
}
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const argument = argv[index]
|
|
const separatorIndex = argument.indexOf('=')
|
|
const optionName = separatorIndex === -1
|
|
? argument
|
|
: argument.slice(0, separatorIndex)
|
|
const inlineValue = separatorIndex === -1
|
|
? null
|
|
: argument.slice(separatorIndex + 1)
|
|
|
|
if (optionName === '--key' || optionName === '--api-key') {
|
|
throw new PublicError(
|
|
'The API key can only be provided through AMAP_WEB_SERVICE_KEY.',
|
|
{ code: 'KEY_ARGUMENT_FORBIDDEN' },
|
|
)
|
|
}
|
|
|
|
if (['--overwrite', '--dry-run', '--help', '--no-types', '--no-keywords'].includes(optionName)) {
|
|
if (inlineValue !== null) {
|
|
throw new PublicError('Boolean options do not accept a value.', {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
if (optionName === '--overwrite')
|
|
config.overwrite = true
|
|
else if (optionName === '--dry-run')
|
|
config.dryRun = true
|
|
else if (optionName === '--help')
|
|
config.help = true
|
|
else if (optionName === '--no-types')
|
|
config.defaultTypesEnabled = false
|
|
else
|
|
config.defaultKeywordsEnabled = false
|
|
continue
|
|
}
|
|
|
|
const valueOptions = new Set([
|
|
'--output',
|
|
'--type-group',
|
|
'--keyword',
|
|
'--page-size',
|
|
'--max-pages',
|
|
'--throttle-ms',
|
|
'--retries',
|
|
'--retry-base-ms',
|
|
'--timeout-ms',
|
|
])
|
|
if (!valueOptions.has(optionName)) {
|
|
throw new PublicError('Unknown option. Run with --help for usage.', {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
const parsed = readOptionValue(argv, index, inlineValue, optionName)
|
|
index = parsed.nextIndex
|
|
|
|
if (optionName === '--output') {
|
|
if (parsed.value.trim() === '')
|
|
throw new PublicError('--output cannot be empty.', { code: 'INVALID_ARGUMENT' })
|
|
config.output = parsed.value
|
|
}
|
|
else if (optionName === '--type-group') {
|
|
config.customTypeGroups.push(validateTypeGroup(parsed.value))
|
|
}
|
|
else if (optionName === '--keyword') {
|
|
const keyword = parsed.value.trim()
|
|
if (keyword === '' || keyword.length > 80) {
|
|
throw new PublicError('--keyword must contain between 1 and 80 characters.', {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
config.customKeywords.push(keyword)
|
|
}
|
|
else if (optionName === '--page-size') {
|
|
config.pageSize = parseInteger(parsed.value, '--page-size', {
|
|
minimum: 1,
|
|
maximum: 25,
|
|
})
|
|
}
|
|
else if (optionName === '--max-pages') {
|
|
config.maxPages = parseInteger(parsed.value, '--max-pages', { minimum: 1 })
|
|
}
|
|
else if (optionName === '--throttle-ms') {
|
|
config.throttleMs = parseInteger(parsed.value, '--throttle-ms', { minimum: 0 })
|
|
}
|
|
else if (optionName === '--retries') {
|
|
config.retries = parseInteger(parsed.value, '--retries', {
|
|
minimum: 0,
|
|
maximum: 10,
|
|
})
|
|
}
|
|
else if (optionName === '--retry-base-ms') {
|
|
config.retryBaseMs = parseInteger(parsed.value, '--retry-base-ms', { minimum: 0 })
|
|
}
|
|
else {
|
|
config.timeoutMs = parseInteger(parsed.value, '--timeout-ms', { minimum: 1 })
|
|
}
|
|
}
|
|
|
|
const maximumPages = Math.ceil(MAX_RESULTS_PER_QUERY / config.pageSize)
|
|
config.maxPages ??= maximumPages
|
|
if (config.maxPages > maximumPages) {
|
|
throw new PublicError(
|
|
`--max-pages cannot exceed ${maximumPages} with the selected page size.`,
|
|
{ code: 'INVALID_ARGUMENT' },
|
|
)
|
|
}
|
|
|
|
const typeGroups = config.customTypeGroups.length > 0
|
|
? config.customTypeGroups
|
|
: config.defaultTypesEnabled
|
|
? DEFAULT_TYPE_GROUPS.map(group => ({ name: group.name, types: [...group.types] }))
|
|
: []
|
|
const keywords = config.customKeywords.length > 0
|
|
? [...new Set(config.customKeywords)]
|
|
: config.defaultKeywordsEnabled
|
|
? [...DEFAULT_KEYWORDS]
|
|
: []
|
|
|
|
if (!config.help && typeGroups.length === 0 && keywords.length === 0) {
|
|
throw new PublicError('At least one type group or keyword query is required.', {
|
|
code: 'INVALID_ARGUMENT',
|
|
})
|
|
}
|
|
|
|
return {
|
|
output: config.output,
|
|
pageSize: config.pageSize,
|
|
maxPages: config.maxPages,
|
|
throttleMs: config.throttleMs,
|
|
retries: config.retries,
|
|
retryBaseMs: config.retryBaseMs,
|
|
timeoutMs: config.timeoutMs,
|
|
overwrite: config.overwrite,
|
|
dryRun: config.dryRun,
|
|
help: config.help,
|
|
typeGroups,
|
|
keywords,
|
|
}
|
|
}
|
|
|
|
export function buildQueryPlan(typeGroups, keywords) {
|
|
const queries = []
|
|
|
|
for (const group of typeGroups) {
|
|
queries.push({
|
|
id: `types:${group.name}`,
|
|
kind: 'types',
|
|
label: group.name,
|
|
types: [...group.types],
|
|
})
|
|
}
|
|
|
|
for (let index = 0; index < keywords.length; index += 1) {
|
|
queries.push({
|
|
id: `keyword:${index + 1}`,
|
|
kind: 'keyword',
|
|
label: keywords[index],
|
|
keyword: keywords[index],
|
|
})
|
|
}
|
|
|
|
return queries
|
|
}
|
|
|
|
function sleep(milliseconds) {
|
|
return new Promise(resolvePromise => setTimeout(resolvePromise, milliseconds))
|
|
}
|
|
|
|
function createThrottler(intervalMs) {
|
|
let lastRequestStartedAt = 0
|
|
|
|
return async function throttle() {
|
|
const elapsed = Date.now() - lastRequestStartedAt
|
|
const remaining = intervalMs - elapsed
|
|
if (remaining > 0)
|
|
await sleep(remaining)
|
|
lastRequestStartedAt = Date.now()
|
|
}
|
|
}
|
|
|
|
export function buildAmapSearchParams(query, pageSize, pageNumber) {
|
|
const searchParams = new URLSearchParams()
|
|
searchParams.set('region', TARGET_ADCODE)
|
|
searchParams.set('city_limit', 'true')
|
|
searchParams.set('page_size', String(pageSize))
|
|
searchParams.set('page_num', String(pageNumber))
|
|
searchParams.set('show_fields', 'business,photos')
|
|
|
|
if (query.kind === 'types')
|
|
searchParams.set('types', query.types.join('|'))
|
|
else
|
|
searchParams.set('keywords', query.keyword)
|
|
|
|
return searchParams
|
|
}
|
|
|
|
function buildRequestUrl(apiKey, query, pageSize, pageNumber) {
|
|
const url = new URL(AMAP_ENDPOINT)
|
|
url.search = buildAmapSearchParams(query, pageSize, pageNumber).toString()
|
|
url.searchParams.set('key', apiKey)
|
|
|
|
return url
|
|
}
|
|
|
|
function isRetryableHttpStatus(status) {
|
|
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
}
|
|
|
|
async function requestAmapPage({
|
|
apiKey,
|
|
query,
|
|
pageSize,
|
|
pageNumber,
|
|
timeoutMs,
|
|
fetchImpl,
|
|
}) {
|
|
const controller = new AbortController()
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs)
|
|
let response
|
|
|
|
try {
|
|
response = await fetchImpl(
|
|
buildRequestUrl(apiKey, query, pageSize, pageNumber),
|
|
{
|
|
method: 'GET',
|
|
headers: { accept: 'application/json' },
|
|
signal: controller.signal,
|
|
},
|
|
)
|
|
}
|
|
catch {
|
|
const timedOut = controller.signal.aborted
|
|
throw new PublicError(
|
|
timedOut ? 'The AMap request timed out.' : 'The AMap request failed at the network layer.',
|
|
{ code: timedOut ? 'TIMEOUT' : 'NETWORK_ERROR', retryable: true },
|
|
)
|
|
}
|
|
finally {
|
|
clearTimeout(timeout)
|
|
}
|
|
|
|
if (!response.ok) {
|
|
throw new PublicError(`AMap returned HTTP ${response.status}.`, {
|
|
code: `HTTP_${response.status}`,
|
|
retryable: isRetryableHttpStatus(response.status),
|
|
})
|
|
}
|
|
|
|
let payload
|
|
try {
|
|
payload = await response.json()
|
|
}
|
|
catch {
|
|
throw new PublicError('AMap returned invalid JSON.', {
|
|
code: 'INVALID_JSON',
|
|
retryable: true,
|
|
})
|
|
}
|
|
|
|
if (!isRecord(payload)) {
|
|
throw new PublicError('AMap returned an invalid response object.', {
|
|
code: 'INVALID_RESPONSE',
|
|
retryable: true,
|
|
})
|
|
}
|
|
|
|
if (String(payload.status) !== '1') {
|
|
const infocode = /^\d+$/.test(String(payload.infocode))
|
|
? String(payload.infocode)
|
|
: 'UNKNOWN'
|
|
throw new PublicError(`AMap rejected the request (infocode ${infocode}).`, {
|
|
code: `AMAP_${infocode}`,
|
|
retryable: RETRYABLE_AMAP_INFOCODES.has(infocode),
|
|
})
|
|
}
|
|
|
|
if (!Array.isArray(payload.pois)) {
|
|
throw new PublicError('AMap response did not contain a POI array.', {
|
|
code: 'INVALID_RESPONSE',
|
|
retryable: true,
|
|
})
|
|
}
|
|
|
|
return payload
|
|
}
|
|
|
|
async function requestWithRetry(options, throttle, onRetry) {
|
|
const { retries, retryBaseMs } = options
|
|
|
|
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
|
await throttle()
|
|
try {
|
|
return await requestAmapPage(options)
|
|
}
|
|
catch (error) {
|
|
const retryable = error instanceof PublicError && error.retryable
|
|
if (!retryable || attempt === retries)
|
|
throw error
|
|
|
|
const delayMs = retryBaseMs * (2 ** attempt) + Math.floor(Math.random() * 250)
|
|
onRetry({ attempt: attempt + 1, retries, delayMs, code: error.code })
|
|
await sleep(delayMs)
|
|
}
|
|
}
|
|
|
|
throw new PublicError('Request retry loop ended unexpectedly.')
|
|
}
|
|
|
|
export async function collectAmapPois(config, {
|
|
apiKey,
|
|
fetchImpl = globalThis.fetch,
|
|
log = console,
|
|
} = {}) {
|
|
if (typeof apiKey !== 'string' || apiKey.trim() === '') {
|
|
throw new PublicError('AMAP_WEB_SERVICE_KEY is required.', {
|
|
code: 'MISSING_KEY',
|
|
})
|
|
}
|
|
if (typeof fetchImpl !== 'function') {
|
|
throw new PublicError('This Node.js runtime does not provide fetch().', {
|
|
code: 'FETCH_UNAVAILABLE',
|
|
})
|
|
}
|
|
|
|
const queries = buildQueryPlan(config.typeGroups, config.keywords)
|
|
const throttle = createThrottler(config.throttleMs)
|
|
const collected = []
|
|
const queryStats = []
|
|
const totals = {
|
|
requests: 0,
|
|
retries: 0,
|
|
received: 0,
|
|
acceptedBeforeDedupe: 0,
|
|
filteredOutByAdcode: 0,
|
|
rejectedInvalid: 0,
|
|
}
|
|
|
|
for (let queryIndex = 0; queryIndex < queries.length; queryIndex += 1) {
|
|
const query = queries[queryIndex]
|
|
const stats = {
|
|
id: query.id,
|
|
kind: query.kind,
|
|
label: query.label,
|
|
pagesFetched: 0,
|
|
received: 0,
|
|
acceptedBeforeDedupe: 0,
|
|
filteredOutByAdcode: 0,
|
|
rejectedInvalid: 0,
|
|
stopReason: null,
|
|
}
|
|
|
|
for (let pageNumber = 1; pageNumber <= config.maxPages; pageNumber += 1) {
|
|
const payload = await requestWithRetry(
|
|
{
|
|
apiKey,
|
|
query,
|
|
pageSize: config.pageSize,
|
|
pageNumber,
|
|
timeoutMs: config.timeoutMs,
|
|
retries: config.retries,
|
|
retryBaseMs: config.retryBaseMs,
|
|
fetchImpl,
|
|
},
|
|
throttle,
|
|
({ attempt, retries, delayMs, code }) => {
|
|
totals.retries += 1
|
|
log.warn(
|
|
`[${queryIndex + 1}/${queries.length}] ${query.id}: transient ${code}; `
|
|
+ `retry ${attempt}/${retries} in ${delayMs} ms.`,
|
|
)
|
|
},
|
|
)
|
|
|
|
totals.requests += 1
|
|
stats.pagesFetched += 1
|
|
stats.received += payload.pois.length
|
|
totals.received += payload.pois.length
|
|
|
|
const normalized = normalizeAmapPage(payload.pois, query)
|
|
stats.acceptedBeforeDedupe += normalized.pois.length
|
|
stats.filteredOutByAdcode += normalized.filteredOutByAdcode
|
|
stats.rejectedInvalid += normalized.rejectedInvalid
|
|
totals.acceptedBeforeDedupe += normalized.pois.length
|
|
totals.filteredOutByAdcode += normalized.filteredOutByAdcode
|
|
totals.rejectedInvalid += normalized.rejectedInvalid
|
|
collected.push(...normalized.pois)
|
|
|
|
log.info(
|
|
`[${queryIndex + 1}/${queries.length}] ${query.id} page ${pageNumber}: `
|
|
+ `received ${payload.pois.length}, accepted ${normalized.pois.length}.`,
|
|
)
|
|
|
|
if (payload.pois.length < config.pageSize) {
|
|
stats.stopReason = 'last-page'
|
|
break
|
|
}
|
|
|
|
if (pageNumber === config.maxPages) {
|
|
const requestedCapacity = pageNumber * config.pageSize
|
|
stats.stopReason = requestedCapacity >= MAX_RESULTS_PER_QUERY
|
|
? 'amap-200-result-cap'
|
|
: 'configured-page-limit'
|
|
}
|
|
}
|
|
|
|
stats.stopReason ??= 'configured-page-limit'
|
|
queryStats.push(stats)
|
|
}
|
|
|
|
const pois = dedupePois(collected)
|
|
return {
|
|
schemaVersion: 1,
|
|
generatedAt: new Date().toISOString(),
|
|
source: {
|
|
provider: 'amap',
|
|
api: 'POI 2.0 text search',
|
|
endpoint: AMAP_ENDPOINT,
|
|
region: TARGET_ADCODE,
|
|
cityLimit: true,
|
|
coordinateSystem: 'GCJ02',
|
|
showFields: ['business', 'photos'],
|
|
},
|
|
reviewStatus: 'unreviewed-candidates',
|
|
notice: 'Candidates require manual verification, content review, and image-rights review before publication.',
|
|
collection: {
|
|
pageSize: config.pageSize,
|
|
maxPagesPerQuery: config.maxPages,
|
|
throttleMs: config.throttleMs,
|
|
queryCount: queries.length,
|
|
queries: queryStats,
|
|
},
|
|
stats: {
|
|
...totals,
|
|
duplicatesMerged: totals.acceptedBeforeDedupe - pois.length,
|
|
uniqueCandidates: pois.length,
|
|
},
|
|
pois,
|
|
}
|
|
}
|
|
|
|
function isPathInside(candidatePath, parentPath) {
|
|
const pathFromParent = relative(parentPath, candidatePath)
|
|
return pathFromParent === ''
|
|
|| (!pathFromParent.startsWith(`..${sep}`) && pathFromParent !== '..')
|
|
}
|
|
|
|
export function assertSafeOutputPath(outputPath, projectRoot = process.cwd()) {
|
|
const absoluteOutputPath = resolve(projectRoot, outputPath)
|
|
for (const directory of PROTECTED_OUTPUT_DIRECTORIES) {
|
|
const protectedPath = resolve(projectRoot, directory)
|
|
if (isPathInside(absoluteOutputPath, protectedPath)) {
|
|
throw new PublicError(
|
|
`Refusing to write generated data inside the ${directory} source directory.`,
|
|
{ code: 'UNSAFE_OUTPUT_PATH' },
|
|
)
|
|
}
|
|
}
|
|
return absoluteOutputPath
|
|
}
|
|
|
|
function defaultOutputPath() {
|
|
const timestamp = new Date().toISOString().replaceAll(':', '-').replaceAll('.', '-')
|
|
return resolve(tmpdir(), `amap-guangming-pois-${timestamp}-${process.pid}.json`)
|
|
}
|
|
|
|
async function writeOutputFile(outputPath, payload, overwrite) {
|
|
await mkdir(dirname(outputPath), { recursive: true })
|
|
await writeFile(
|
|
outputPath,
|
|
`${JSON.stringify(payload, null, 2)}\n`,
|
|
{
|
|
encoding: 'utf8',
|
|
flag: overwrite ? 'w' : 'wx',
|
|
mode: 0o600,
|
|
},
|
|
)
|
|
}
|
|
|
|
function describeQuery(query) {
|
|
return query.kind === 'types'
|
|
? `${query.id} (${query.types.join('|')})`
|
|
: `${query.id} (${query.keyword})`
|
|
}
|
|
|
|
export async function main(argv = process.argv.slice(2), env = process.env) {
|
|
const config = parseCliArgs(argv)
|
|
if (config.help) {
|
|
console.log(HELP_TEXT.trim())
|
|
return
|
|
}
|
|
|
|
const queries = buildQueryPlan(config.typeGroups, config.keywords)
|
|
if (config.dryRun) {
|
|
console.log(`Region: ${TARGET_ADCODE}; city_limit=true`)
|
|
console.log(`Queries: ${queries.length}; up to ${config.maxPages} pages each`)
|
|
queries.forEach(query => console.log(`- ${describeQuery(query)}`))
|
|
return
|
|
}
|
|
|
|
const apiKey = env.AMAP_WEB_SERVICE_KEY
|
|
if (typeof apiKey !== 'string' || apiKey.trim() === '') {
|
|
throw new PublicError('AMAP_WEB_SERVICE_KEY is required.', {
|
|
code: 'MISSING_KEY',
|
|
})
|
|
}
|
|
|
|
const outputPath = config.output === null
|
|
? defaultOutputPath()
|
|
: assertSafeOutputPath(config.output)
|
|
const result = await collectAmapPois(config, { apiKey })
|
|
await writeOutputFile(outputPath, result, config.overwrite)
|
|
|
|
console.log(`Saved ${result.stats.uniqueCandidates} unique candidates to ${outputPath}`)
|
|
const cappedQueries = result.collection.queries
|
|
.filter(query => query.stopReason !== 'last-page')
|
|
if (cappedQueries.length > 0) {
|
|
console.warn(
|
|
`${cappedQueries.length} queries ended at a result/page cap; split them into narrower queries if fuller recall is required.`,
|
|
)
|
|
}
|
|
}
|
|
|
|
function publicErrorMessage(error) {
|
|
const fallback = 'Unexpected failure.'
|
|
const message = error instanceof Error ? error.message : fallback
|
|
const apiKey = process.env.AMAP_WEB_SERVICE_KEY
|
|
if (typeof apiKey !== 'string' || apiKey === '')
|
|
return message
|
|
return message.replaceAll(apiKey, '[REDACTED]')
|
|
}
|
|
|
|
const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : null
|
|
if (invokedPath === import.meta.url) {
|
|
main().catch((error) => {
|
|
console.error(`Error: ${publicErrorMessage(error)}`)
|
|
process.exitCode = 1
|
|
})
|
|
}
|