Tony/node-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: Tony:main
Branches Tags
Tony:main
Tony:dev
...
pull from: Tony:dev
Branches Tags
Tony:main
Tony:dev

1 Commits
main ... dev

Author SHA1 Message Date
wangyilan
5bfcc8c114 feat:地名搜索缓存功能 2024-09-06 17:32:38 +08:00
4 changed files with 218 additions and 73 deletions
Show Stats Expand all files Collapse all files

41
src/tdtmap/data/coordinates.ts Normal file
View File

@ -0,0 +1,41 @@
// import { generateUUID } from "@/utils"
const document = [
{
id: 1,
content: '明月村',
meta: {
address: "重庆市涪陵区",
lonlat: "107.039584,29.467733",
name: "明月村"
}
},
{
id:2,
content:'双桥村',
meta:{
address:"重庆市酉阳县",
lonlat:"108.695672,28.6984455",
name:"双桥村"
}
},
{
id:3,
content:'梧桐村',
meta:{
address:"重庆市万州区",
lonlat:"108.703826,30.6362342",
name:"梧桐村"
}
},
]
export const searchDoc = (query:string) => {
return document.filter(item => {
const queryArr = query.split(" ");
const existMatchDoc = queryArr.find(_query => {
return item.content.indexOf(_query) !== -1
})
return !!existMatchDoc;
})
}

115
src/tdtmap/tdtmap.controller.ts
View File

@ -1,66 +1,87 @@
import { Controller, Get, Query, Logger, Res, BadRequestException, HttpStatus, StreamableFile } from '@nestjs/common';
import {
Controller,
Get,
Query,
Logger,
Res,
BadRequestException,
HttpStatus,
StreamableFile,
} from '@nestjs/common';
import { TdtmapService } from './tdtmap.service';
import { LRUCache } from 'lru-cache';
import { BloomFilter } from 'bloom-filters';
import { Public } from '../common/public.guard';
import axios from 'axios'
import axios from 'axios';
import { Response } from 'express';
// 热点缓存
const cacheMap = new LRUCache<string, ArrayBufferView>({
max: 20000, // 最大缓存项目数
ttl: 1000 * 60 * 60 * 24 * 5, // 缓存时间5天，
})
});
// 布隆过滤器
const bloomFilter = new Set<string>()
const bloomFilter = new Set<string>();
const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`
const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}`
const urlFormatter = (
x: number,
y: number,
z: number,
type: string = 'vec_w',
token: string,
) =>
`http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`;
const mapUrlFormatter = (x: number, y: number, z: number, type: string) =>
`X_${x}&Y_${y}&Z_${z}&type_${type}`;
@Controller('tdtmap')
@Public()
export class TdtmapController {
constructor(
private readonly tdtmapService: TdtmapService
) {
this.initBloomFilter()
constructor(private readonly tdtmapService: TdtmapService) {
this.initBloomFilter();
}
private readonly logger: Logger = new Logger(TdtmapController.name)
deleteAllMinioObj(tile_name:string) {
this.tdtmapService.deleteMinioObj(tile_name)
private readonly logger: Logger = new Logger(TdtmapController.name);
deleteAllMinioObj(tile_name: string) {
this.tdtmapService.deleteMinioObj(tile_name);
}
// 初始化过滤器
async initBloomFilter() {
const minioObjNamesArr = await this.tdtmapService.getTilesNameList().catch(err => {
this.logger.error(`get bucket obj names failed!`)
})
const minioObjNamesArr = await this.tdtmapService
.getTilesNameList()
.catch((err) => {
this.logger.error(`get bucket obj names failed!`);
});
if (!Array.isArray(minioObjNamesArr)) return;
minioObjNamesArr.forEach(item => { bloomFilter.add(item); })
this.logger.debug(`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`)
minioObjNamesArr.forEach((item) => {
bloomFilter.add(item);
});
this.logger.debug(
`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`,
);
}
@Get("/tile")
@Get('/tile')
async getTile(
@Res({ passthrough: true }) response:Response,
@Query('x') x:number,
@Query('y') y:number,
@Query('l') l:number,
@Query('T') type:string,
@Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e'
@Res({ passthrough: true }) response: Response,
@Query('x') x: number,
@Query('y') y: number,
@Query('l') l: number,
@Query('T') type: string,
@Query('tk') tk: string = '6988fa4ec7ca5ed400097b9bf9dfc22e',
) {
// 检查参数
if (!x || !y || !l || !type || !tk) return;
const tileFormattedName = mapUrlFormatter(x, y, l, type);
response.setHeader("Content-Type", "image/png"); // 设置文件请求头
response.setHeader('Content-Type', 'image/png'); // 设置文件请求头
// 查找缓存
const cacheItem:any = cacheMap.get(tileFormattedName);
const cacheItem: any = cacheMap.get(tileFormattedName);
// 获取到缓存中的内容， 直接返回
// @ts-ignore
if (cacheItem && ArrayBuffer.isView(cacheItem)) {
Buffer.from(cacheItem as any, 'utf-8')
Buffer.from(cacheItem as any, 'utf-8');
return new StreamableFile(cacheItem as any); // 请求成功
}
@ -68,29 +89,43 @@ export class TdtmapController {
const existInDB = bloomFilter.has(tileFormattedName);
if (existInDB) {
// minio 中存在，从 minio 取出
const dbItem:Buffer = await this.tdtmapService.getTileFromLocal(tileFormattedName) // 从 minio 取出来
cacheMap.set(tileFormattedName, dbItem)
Buffer.from(dbItem as any, 'utf-8')
const dbItem: Buffer =
await this.tdtmapService.getTileFromLocal(tileFormattedName); // 从 minio 取出来
cacheMap.set(tileFormattedName, dbItem);
Buffer.from(dbItem as any, 'utf-8');
return new StreamableFile(dbItem);
} else {
// 如果本地数据库都不存在
const res = await axios({
url: urlFormatter(x, y, l, type, tk),
method: "GET",
responseType: "arraybuffer"
method: 'GET',
responseType: 'arraybuffer',
}).catch((err) => {
this.logger.error(`Tile req Fail => ${err}`)
})
this.logger.error(`Tile req Fail => ${err}`);
});
// 写入缓存和本地
if (res && typeof res.data === 'object') {
cacheMap.set(tileFormattedName, res.data)
const saveSuccess = await this.tdtmapService.setTileToLocal(tileFormattedName, res.data);
cacheMap.set(tileFormattedName, res.data);
const saveSuccess = await this.tdtmapService.setTileToLocal(
tileFormattedName,
res.data,
);
if (saveSuccess) bloomFilter.add(tileFormattedName);
Buffer.from(res.data, 'utf-8')
Buffer.from(res.data, 'utf-8');
return new StreamableFile(res.data); // 请求成功
} else {
throw new BadRequestException("request failed!"); // 请求失败
throw new BadRequestException('request failed!'); // 请求失败
}
}
}
//搜索
@Get('/search')
async searchMapData(@Query('q') query: string) {
try {
return await this.tdtmapService.searchMapData(query); // 调用服务层
} catch (error) {
return { error: error.message };
}
}
}

7
src/tdtmap/tdtmap.module.ts
View File

@ -1,8 +1,15 @@
import { Module } from '@nestjs/common';
import { CacheModule } from '@nestjs/cache-manager';
import { TdtmapService } from './tdtmap.service';
import { TdtmapController } from './tdtmap.controller';
@Module({
imports: [
CacheModule.register({
ttl: 300, // 设置缓存过期时间
max: 100, // 设置最大缓存数
}),
],
controllers: [TdtmapController],
providers: [TdtmapService],
})

128
src/tdtmap/tdtmap.service.ts
View File

@ -1,14 +1,18 @@
import { Injectable, Logger } from '@nestjs/common';
import { Injectable, Logger, Inject } from '@nestjs/common';
import * as Minio from 'minio';
import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { Cache } from 'cache-manager';
import axios from 'axios';
import { searchDoc } from './data/coordinates';
@Injectable()
export class TdtmapService {
private readonly minioClient: Minio.Client;
private readonly logger: Logger = new Logger(TdtmapService.name)
constructor() {
private readonly logger: Logger = new Logger(TdtmapService.name);
constructor(@Inject(CACHE_MANAGER) private cacheManager: Cache) {
this.minioClient = new Minio.Client({
endPoint: '117.73.12.97',
port: 9000,
port: 9000,
useSSL: false,
accessKey: 'inspur',
secretKey: 'inspur123',
@ -16,42 +20,100 @@ export class TdtmapService {
}
setMinioObj() {}
async getTilesNameList():Promise<string[]> {
async getTilesNameList(): Promise<string[]> {
return new Promise(async (resolve, reject) => {
const stream = await this.minioClient.listObjectsV2("nestfiles");
const res:string[] = []
stream.on("data", (obj) => { res.push(obj.name) })
stream.on("end", () => { resolve(res) })
stream.on("error", () => { reject() })
})
}
async getTileFromLocal(tile_name:string):Promise<Buffer> {
return new Promise(async (resolve, reject) => {
const stream = await this.minioClient.getObject("nestfiles", tile_name)
const res:any[] = []
stream.on("data", (data:Buffer) => {
if (Buffer.isBuffer(data)) res.push(data)
})
stream.on("end", () => {
if (res.length > 0) resolve(Buffer.concat(res))
else reject()
})
stream.on("error", () => { reject() })
})
const stream = await this.minioClient.listObjectsV2('nestfiles');
const res: string[] = [];
stream.on('data', (obj) => {
res.push(obj.name);
});
stream.on('end', () => {
resolve(res);
});
stream.on('error', () => {
reject();
});
});
}
async getTileFromLocal(tile_name: string): Promise<Buffer> {
return new Promise(async (resolve, reject) => {
const stream = await this.minioClient.getObject('nestfiles', tile_name);
const res: any[] = [];
stream.on('data', (data: Buffer) => {
if (Buffer.isBuffer(data)) res.push(data);
});
stream.on('end', () => {
if (res.length > 0) resolve(Buffer.concat(res));
else reject();
});
stream.on('error', () => {
reject();
});
});
}
async setTileToLocal(tile_name:string, data:Buffer):Promise<boolean> {
const res = await this.minioClient.putObject("nestfiles", tile_name, data).catch((err) => {
this.logger.error(`Minio Save Obj failed! => ${err}`)
})
async setTileToLocal(tile_name: string, data: Buffer): Promise<boolean> {
const res = await this.minioClient
.putObject('nestfiles', tile_name, data)
.catch((err) => {
this.logger.error(`Minio Save Obj failed! => ${err}`);
});
if (!res) return false;
return true;
}
deleteMinioObj(tile_name:string) {
this.minioClient.removeObject("nestfiles", tile_name);
deleteMinioObj(tile_name: string) {
this.minioClient.removeObject('nestfiles', tile_name);
}
//地图缓存
private readonly tdSearchUrl = 'https://api.tianditu.gov.cn/v2/search';
async searchMapData(query: string): Promise<any> {
//从缓存请求数据
const cacheData = await this.cacheManager.get(query);
if (cacheData) {
return cacheData;
}
//缓存无数据；从外部请求数据
try {
const posInputVal = '';
const tdSearchResponse = await axios.get(
'https://api.tianditu.gov.cn/v2/search',
{
params: {
type: 'query',
postStr: JSON.stringify({
yingjiType: 1,
sourceType: 0,
keyWord: posInputVal,
level: 18,
mapBound: '73.66, 3.86, 135.05, 53.55',
queryType: '4',
start: 0,
queryTerminal: 10000,
}),
tk: '3499364c33fd4aa4415dd8765d4c5b77',
},
headers: {},
},
);
//保存外部数据到缓存中
if (tdSearchResponse.data && tdSearchResponse.data.length > 0) {
await this.cacheManager.set(query, tdSearchResponse);
return tdSearchResponse;
}
} catch (error) {
console.error('从外部请求失败', error.message);
}
//外部请求失败时从本地查找
const localSearchResult = searchDoc(query);
if (localSearchResult.length > 0) {
//写入缓存
await this.cacheManager.set(query, localSearchResult);
return localSearchResult;
} else {
console.log('未找到相关数据');
}
}
}