feat:地名搜索缓存功能

This commit is contained in:
wangyilan 2024-09-06 17:32:38 +08:00
parent 0c65c8f0d4
commit 5bfcc8c114
4 changed files with 218 additions and 73 deletions

View File

@ -0,0 +1,41 @@
// import { generateUUID } from "@/utils"
const document = [
{
id: 1,
content: '明月村',
meta: {
address: "重庆市涪陵区",
lonlat: "107.039584,29.467733",
name: "明月村"
}
},
{
id:2,
content:'双桥村',
meta:{
address:"重庆市酉阳县",
lonlat:"108.695672,28.6984455",
name:"双桥村"
}
},
{
id:3,
content:'梧桐村',
meta:{
address:"重庆市万州区",
lonlat:"108.703826,30.6362342",
name:"梧桐村"
}
},
]
export const searchDoc = (query:string) => {
return document.filter(item => {
const queryArr = query.split(" ");
const existMatchDoc = queryArr.find(_query => {
return item.content.indexOf(_query) !== -1
})
return !!existMatchDoc;
})
}

View File

@ -1,66 +1,87 @@
import { Controller, Get, Query, Logger, Res, BadRequestException, HttpStatus, StreamableFile } from '@nestjs/common'; import {
Controller,
Get,
Query,
Logger,
Res,
BadRequestException,
HttpStatus,
StreamableFile,
} from '@nestjs/common';
import { TdtmapService } from './tdtmap.service'; import { TdtmapService } from './tdtmap.service';
import { LRUCache } from 'lru-cache'; import { LRUCache } from 'lru-cache';
import { BloomFilter } from 'bloom-filters'; import { BloomFilter } from 'bloom-filters';
import { Public } from '../common/public.guard'; import { Public } from '../common/public.guard';
import axios from 'axios' import axios from 'axios';
import { Response } from 'express'; import { Response } from 'express';
// 热点缓存 // 热点缓存
const cacheMap = new LRUCache<string, ArrayBufferView>({ const cacheMap = new LRUCache<string, ArrayBufferView>({
max: 20000, // 最大缓存项目数 max: 20000, // 最大缓存项目数
ttl: 1000 * 60 * 60 * 24 * 5, // 缓存时间5天 ttl: 1000 * 60 * 60 * 24 * 5, // 缓存时间5天
}) });
// 布隆过滤器 // 布隆过滤器
const bloomFilter = new Set<string>() const bloomFilter = new Set<string>();
const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}` const urlFormatter = (
const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}` x: number,
y: number,
z: number,
type: string = 'vec_w',
token: string,
) =>
`http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`;
const mapUrlFormatter = (x: number, y: number, z: number, type: string) =>
`X_${x}&Y_${y}&Z_${z}&type_${type}`;
@Controller('tdtmap') @Controller('tdtmap')
@Public() @Public()
export class TdtmapController { export class TdtmapController {
constructor( constructor(private readonly tdtmapService: TdtmapService) {
private readonly tdtmapService: TdtmapService this.initBloomFilter();
) {
this.initBloomFilter()
} }
private readonly logger: Logger = new Logger(TdtmapController.name) private readonly logger: Logger = new Logger(TdtmapController.name);
deleteAllMinioObj(tile_name:string) { deleteAllMinioObj(tile_name: string) {
this.tdtmapService.deleteMinioObj(tile_name) this.tdtmapService.deleteMinioObj(tile_name);
} }
// 初始化过滤器 // 初始化过滤器
async initBloomFilter() { async initBloomFilter() {
const minioObjNamesArr = await this.tdtmapService.getTilesNameList().catch(err => { const minioObjNamesArr = await this.tdtmapService
this.logger.error(`get bucket obj names failed!`) .getTilesNameList()
}) .catch((err) => {
this.logger.error(`get bucket obj names failed!`);
});
if (!Array.isArray(minioObjNamesArr)) return; if (!Array.isArray(minioObjNamesArr)) return;
minioObjNamesArr.forEach(item => { bloomFilter.add(item); }) minioObjNamesArr.forEach((item) => {
this.logger.debug(`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`) bloomFilter.add(item);
});
this.logger.debug(
`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`,
);
} }
@Get("/tile") @Get('/tile')
async getTile( async getTile(
@Res({ passthrough: true }) response:Response, @Res({ passthrough: true }) response: Response,
@Query('x') x:number, @Query('x') x: number,
@Query('y') y:number, @Query('y') y: number,
@Query('l') l:number, @Query('l') l: number,
@Query('T') type:string, @Query('T') type: string,
@Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e' @Query('tk') tk: string = '6988fa4ec7ca5ed400097b9bf9dfc22e',
) { ) {
// 检查参数 // 检查参数
if (!x || !y || !l || !type || !tk) return; if (!x || !y || !l || !type || !tk) return;
const tileFormattedName = mapUrlFormatter(x, y, l, type); const tileFormattedName = mapUrlFormatter(x, y, l, type);
response.setHeader("Content-Type", "image/png"); // 设置文件请求头 response.setHeader('Content-Type', 'image/png'); // 设置文件请求头
// 查找缓存 // 查找缓存
const cacheItem:any = cacheMap.get(tileFormattedName); const cacheItem: any = cacheMap.get(tileFormattedName);
// 获取到缓存中的内容, 直接返回 // 获取到缓存中的内容, 直接返回
// @ts-ignore // @ts-ignore
if (cacheItem && ArrayBuffer.isView(cacheItem)) { if (cacheItem && ArrayBuffer.isView(cacheItem)) {
Buffer.from(cacheItem as any, 'utf-8') Buffer.from(cacheItem as any, 'utf-8');
return new StreamableFile(cacheItem as any); // 请求成功 return new StreamableFile(cacheItem as any); // 请求成功
} }
@ -68,29 +89,43 @@ export class TdtmapController {
const existInDB = bloomFilter.has(tileFormattedName); const existInDB = bloomFilter.has(tileFormattedName);
if (existInDB) { if (existInDB) {
// minio 中存在,从 minio 取出 // minio 中存在,从 minio 取出
const dbItem:Buffer = await this.tdtmapService.getTileFromLocal(tileFormattedName) // 从 minio 取出来 const dbItem: Buffer =
cacheMap.set(tileFormattedName, dbItem) await this.tdtmapService.getTileFromLocal(tileFormattedName); // 从 minio 取出来
Buffer.from(dbItem as any, 'utf-8') cacheMap.set(tileFormattedName, dbItem);
Buffer.from(dbItem as any, 'utf-8');
return new StreamableFile(dbItem); return new StreamableFile(dbItem);
} else { } else {
// 如果本地数据库都不存在 // 如果本地数据库都不存在
const res = await axios({ const res = await axios({
url: urlFormatter(x, y, l, type, tk), url: urlFormatter(x, y, l, type, tk),
method: "GET", method: 'GET',
responseType: "arraybuffer" responseType: 'arraybuffer',
}).catch((err) => { }).catch((err) => {
this.logger.error(`Tile req Fail => ${err}`) this.logger.error(`Tile req Fail => ${err}`);
}) });
// 写入缓存和本地 // 写入缓存和本地
if (res && typeof res.data === 'object') { if (res && typeof res.data === 'object') {
cacheMap.set(tileFormattedName, res.data) cacheMap.set(tileFormattedName, res.data);
const saveSuccess = await this.tdtmapService.setTileToLocal(tileFormattedName, res.data); const saveSuccess = await this.tdtmapService.setTileToLocal(
tileFormattedName,
res.data,
);
if (saveSuccess) bloomFilter.add(tileFormattedName); if (saveSuccess) bloomFilter.add(tileFormattedName);
Buffer.from(res.data, 'utf-8') Buffer.from(res.data, 'utf-8');
return new StreamableFile(res.data); // 请求成功 return new StreamableFile(res.data); // 请求成功
} else { } else {
throw new BadRequestException("request failed!"); // 请求失败 throw new BadRequestException('request failed!'); // 请求失败
} }
} }
} }
//搜索
@Get('/search')
async searchMapData(@Query('q') query: string) {
try {
return await this.tdtmapService.searchMapData(query); // 调用服务层
} catch (error) {
return { error: error.message };
}
}
} }

View File

@ -1,8 +1,15 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { CacheModule } from '@nestjs/cache-manager';
import { TdtmapService } from './tdtmap.service'; import { TdtmapService } from './tdtmap.service';
import { TdtmapController } from './tdtmap.controller'; import { TdtmapController } from './tdtmap.controller';
@Module({ @Module({
imports: [
CacheModule.register({
ttl: 300, // 设置缓存过期时间
max: 100, // 设置最大缓存数
}),
],
controllers: [TdtmapController], controllers: [TdtmapController],
providers: [TdtmapService], providers: [TdtmapService],
}) })

View File

@ -1,11 +1,15 @@
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger, Inject } from '@nestjs/common';
import * as Minio from 'minio'; import * as Minio from 'minio';
import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { Cache } from 'cache-manager';
import axios from 'axios';
import { searchDoc } from './data/coordinates';
@Injectable() @Injectable()
export class TdtmapService { export class TdtmapService {
private readonly minioClient: Minio.Client; private readonly minioClient: Minio.Client;
private readonly logger: Logger = new Logger(TdtmapService.name) private readonly logger: Logger = new Logger(TdtmapService.name);
constructor() { constructor(@Inject(CACHE_MANAGER) private cacheManager: Cache) {
this.minioClient = new Minio.Client({ this.minioClient = new Minio.Client({
endPoint: '117.73.12.97', endPoint: '117.73.12.97',
port: 9000, port: 9000,
@ -16,42 +20,100 @@ export class TdtmapService {
} }
setMinioObj() {} setMinioObj() {}
async getTilesNameList(): Promise<string[]> {
async getTilesNameList():Promise<string[]> {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const stream = await this.minioClient.listObjectsV2("nestfiles"); const stream = await this.minioClient.listObjectsV2('nestfiles');
const res:string[] = [] const res: string[] = [];
stream.on("data", (obj) => { res.push(obj.name) }) stream.on('data', (obj) => {
stream.on("end", () => { resolve(res) }) res.push(obj.name);
stream.on("error", () => { reject() }) });
}) stream.on('end', () => {
resolve(res);
});
stream.on('error', () => {
reject();
});
});
} }
async getTileFromLocal(tile_name:string):Promise<Buffer> { async getTileFromLocal(tile_name: string): Promise<Buffer> {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
const stream = await this.minioClient.getObject("nestfiles", tile_name) const stream = await this.minioClient.getObject('nestfiles', tile_name);
const res:any[] = [] const res: any[] = [];
stream.on("data", (data:Buffer) => { stream.on('data', (data: Buffer) => {
if (Buffer.isBuffer(data)) res.push(data) if (Buffer.isBuffer(data)) res.push(data);
}) });
stream.on("end", () => { stream.on('end', () => {
if (res.length > 0) resolve(Buffer.concat(res)) if (res.length > 0) resolve(Buffer.concat(res));
else reject() else reject();
}) });
stream.on("error", () => { reject() }) stream.on('error', () => {
}) reject();
});
});
} }
async setTileToLocal(tile_name: string, data: Buffer): Promise<boolean> {
async setTileToLocal(tile_name:string, data:Buffer):Promise<boolean> { const res = await this.minioClient
const res = await this.minioClient.putObject("nestfiles", tile_name, data).catch((err) => { .putObject('nestfiles', tile_name, data)
this.logger.error(`Minio Save Obj failed! => ${err}`) .catch((err) => {
}) this.logger.error(`Minio Save Obj failed! => ${err}`);
});
if (!res) return false; if (!res) return false;
return true; return true;
} }
deleteMinioObj(tile_name:string) { deleteMinioObj(tile_name: string) {
this.minioClient.removeObject("nestfiles", tile_name); this.minioClient.removeObject('nestfiles', tile_name);
}
//地图缓存
private readonly tdSearchUrl = 'https://api.tianditu.gov.cn/v2/search';
async searchMapData(query: string): Promise<any> {
//从缓存请求数据
const cacheData = await this.cacheManager.get(query);
if (cacheData) {
return cacheData;
}
//缓存无数据;从外部请求数据
try {
const posInputVal = '';
const tdSearchResponse = await axios.get(
'https://api.tianditu.gov.cn/v2/search',
{
params: {
type: 'query',
postStr: JSON.stringify({
yingjiType: 1,
sourceType: 0,
keyWord: posInputVal,
level: 18,
mapBound: '73.66, 3.86, 135.05, 53.55',
queryType: '4',
start: 0,
queryTerminal: 10000,
}),
tk: '3499364c33fd4aa4415dd8765d4c5b77',
},
headers: {},
},
);
//保存外部数据到缓存中
if (tdSearchResponse.data && tdSearchResponse.data.length > 0) {
await this.cacheManager.set(query, tdSearchResponse);
return tdSearchResponse;
}
} catch (error) {
console.error('从外部请求失败', error.message);
}
//外部请求失败时从本地查找
const localSearchResult = searchDoc(query);
if (localSearchResult.length > 0) {
//写入缓存
await this.cacheManager.set(query, localSearchResult);
return localSearchResult;
} else {
console.log('未找到相关数据');
}
} }
} }