跳转至

go importer 相关指令

状态: 📝 草稿

创建日期: 2026-02-28 最后更新: 2026-02-28

临时存储

contract 设计文档的 topic（Claude 会话恢复）：

claude --resume 755378b1-041e-43f2-afc0-13f1334feb10

操作相关

日常运行

go run cmd/importer/main.go --config_dir ~/data/real/goimporter/configs --task transform --board_id TST

执行初始化

go run cmd/importer/main.go --config_dir ~/data/real/goimporter/configs --task transform --board_id TST --init

go run cmd/importer/main.go --config_dir ~/goimporter/configs --task transform --board_id TST --init

go run cmd/importer/main.go --config_dir ~/goimporter/configs --task transform --board_id TST --query '{}'

文件同步

rsync -avz --delete --progress \
  --filter=':- .gitignore' \
  --include '*/' --include '*.go' --include '*.sh' \
  --include '*.toml' --include '*.sum' --include '*.mod' \
  --exclude '*' \
  /Users/pengxin/data/real/goimporter/src/ cab5:~/goimporter/src

rsync -avz --delete --progress \
  --filter=':- .gitignore' \
  --include '*/' --include '*.go' --include '*.sh' \
  --include '*.toml' --include '*.sum' --include '*.mod' \
  --exclude '*' \
  /Users/pengxin/data/real/gowatch/ cab5:~/gowatch

服务器上执行 batch

go run cmd/batch/main.go migrate_boundary --config_dir ~/data/real/goimporter/configs --limit 20 --dry_run

初始化并 watch

无日志：

nohup ~/goimporter/src/bin/importer --config_dir ~/goimporter/configs --task transform --board_id EDM --init > /dev/null 2>&1 &

带日志：

# RM1
nohup ~/goimporter/src/bin/importer --config_dir ~/goimporter/configs --task transform --board_id RM1 --init >> ~/goimporter/logs/init-RM1.log 2>&1 &

# EDM
nohup ~/goimporter/src/bin/importer --config_dir ~/goimporter/configs --task transform --board_id EDM --init >> ~/goimporter/logs/init-EDM.log 2>&1 &

# BRE
nohup ~/goimporter/src/bin/importer --config_dir ~/goimporter/configs --task transform --board_id BRE --init >> ~/goimporter/logs/init-BRE.log 2>&1 &

查看进程 / 日志

# 查看进程是否在运行
ps aux | grep importer

# 实时查看某个 board 的日志
tail -f ~/goimporter/logs/init-RM1.log

DB 相关

触发 watch（更新数据库内容）

db.reso_test_evow_merged.updateOne(
  { "_id": "TST001" },
  { $set: { "TaxYear": Math.floor(Math.random() * (2026 - 1990 + 1)) + 1990 } }
);

db.reso_test_evow_merged.updateOne(
  { "_id": "TRBX12240721" },
  { $set: { "TaxYear": Math.floor(Math.random() * (2026 - 1990 + 1)) + 1990 } }
);

db.reso_test_evow_merged.updateMany(
  { },
  { $set: { "TaxYear": Math.floor(Math.random() * (2026 - 1990 + 1)) + 1990 } }
);

db.listing_tst_rm.drop(); db.listing_tst_src.drop();

备份某个集合

db.boundary.aggregate([{ $match: {} }, { $out: "boundary_backup_20251210" }]);

删除集合内的数据

db.boundary.deleteMany({})

重置若干个表

use importer

["failed_documents", "geo_warning", "listing_bre_rm", "listing_bre_src",
 "listing_car_rm", "listing_car_src", "listing_test_board_2_rm", "listing_test_board_2_src",
 "listing_test_board_rm", "listing_test_board_src", "listing_trb_rm", "listing_trb_src",
 "listing_tst_rm", "listing_tst_src", "building", "property",
 "listing_clg_rm", "listing_clg_src", "listing_ddf_rm", "listing_ddf_src",
 "listing_edm_rm", "listing_edm_src", "listing_otw_rm", "listing_otw_src",
 "listing_rhb_rm", "listing_rhb_src", "listing_rm1_rm", "listing_rm1_src"
].forEach(c => print(db[c].drop() ? `Success: ${c}` : `Failed/Not Found: ${c}`));

查询 building 对应的 listing

db.listing_rm.aggregate([
  { $match: { "addr_calc.building_id": { $gt: 0 } } },
  { $group: { _id: "$addr_calc.building_id", count: { $sum: 1 } } },
  { $match: { count: { $gt: 2 } } },
  { $sort: { count: -1 } },
  { $lookup: { from: "building", localField: "_id", foreignField: "_id", as: "b" } },
  { $project: { count: 1, uaddr: { $arrayElemAt: ["$b.uaddr", 0] } } }
])

数据导入导出

# 导出
mongodump --uri "mongodb://d1:d1@ca12:27017/listing?authSource=admin&ssl=true&tlsInsecure=true" --collection=geo_cache --gzip
mongodump --uri "mongodb://d1:d1@ca12:27017/listing?authSource=admin&ssl=true&tlsInsecure=true" --collection=boundary --gzip
mongodump --uri "mongodb://d1:d1@ca12:27017/listing?authSource=admin&ssl=true&tlsInsecure=true" --collection=census2021 --gzip

scp -r dump/listing/census2021.* 167.114.208.35:~/dump/listing/

# 导入
mongorestore "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/tmp?authSource=admin" --db tmp --collection census2021 --drop --gzip dump/listing/census2021.bson.gz
mongorestore "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/tmp?authSource=admin" --db tmp --collection geo_cache --drop --gzip dump/listing/geo_cache.bson.gz
mongorestore "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/tmp?authSource=admin" --db tmp --collection boundary --drop --gzip dump/listing/boundary.bson.gz

一个导出导入的例子

mongodump --uri "mongodb://d1:d1@ca12:27017/data?authSource=admin&ssl=true&tlsInsecure=true" --collection=zip --gzip
mongorestore "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/importer?authSource=admin" --collection geo_zip --drop --gzip zip.bson.gz

备份表 cab4

mongodump --uri "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/importer?authSource=admin" --collection boundary --gzip
mv dump/importer/boundary.* 01_db_importer_backup/

转移表到新的库

mongodump --uri "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/importer?authSource=admin" --collection geo_zip --gzip
mongorestore "mongodb://cab4user:PvUZy5kwqrfN@127.0.0.1:27017/tmp?authSource=admin" --collection geo_zip --drop --gzip dump/importer/geo_zip.bson.gz