From 7c117d0440715991ae8c9ce26d5013609cfa176a Mon Sep 17 00:00:00 2001 From: Tony Date: Wed, 4 Sep 2024 14:49:05 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A4=A9=E5=9C=B0=E5=9B=BETile?= =?UTF-8?q?=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tdtmap/entities/tdtmap.entity.ts | 2 +- src/tdtmap/entities/tdttile.entity.ts | 19 ----------- src/tdtmap/tdtmap.controller.ts | 35 ++++++++++++++------- src/tdtmap/tdtmap.service.ts | 45 +++++++++++++++++++++------ 4 files changed, 61 insertions(+), 40 deletions(-) delete mode 100644 src/tdtmap/entities/tdttile.entity.ts diff --git a/src/tdtmap/entities/tdtmap.entity.ts b/src/tdtmap/entities/tdtmap.entity.ts index 0535e36..fbebfcc 100644 --- a/src/tdtmap/entities/tdtmap.entity.ts +++ b/src/tdtmap/entities/tdtmap.entity.ts @@ -1,7 +1,7 @@ import { Column, Entity, PrimaryGeneratedColumn } from "typeorm"; @Entity("tdt_local") -export class Tdtmap { +export class TdtLocal { @PrimaryGeneratedColumn() id: number; diff --git a/src/tdtmap/entities/tdttile.entity.ts b/src/tdtmap/entities/tdttile.entity.ts deleted file mode 100644 index b9214bc..0000000 --- a/src/tdtmap/entities/tdttile.entity.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Column, Entity, PrimaryGeneratedColumn } from "typeorm"; - -@Entity("tdt_tile") -export class Tdtmap { - @PrimaryGeneratedColumn() - id: number; - - @Column({ type: 'varchar', length: 120, nullable: false }) - url: string; - - @Column({ type: 'varchar', length: 120, nullable: false }) - name: string; - - @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) - create_time: Date; - - @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) - update_time: Date; -} diff --git a/src/tdtmap/tdtmap.controller.ts b/src/tdtmap/tdtmap.controller.ts index e72191f..27276cc 100644 --- a/src/tdtmap/tdtmap.controller.ts +++ b/src/tdtmap/tdtmap.controller.ts @@ -13,7 +13,7 @@ const cacheMap = new LRUCache({ }) // 布隆过滤器 -const bloomFilter = new BloomFilter(10, 4) +const bloomFilter = new Set() const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}` const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}` @@ -23,20 +23,34 @@ const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${ export class TdtmapController { constructor( private readonly tdtmapService: TdtmapService - ) {} + ) { + this.initBloomFilter() + } private readonly logger: Logger = new Logger(TdtmapController.name) + deleteAllMinioObj(tile_name:string) { + this.tdtmapService.deleteMinioObj(tile_name) + } + // 初始化过滤器 + async initBloomFilter() { + const minioObjNamesArr = await this.tdtmapService.getTilesNameList().catch(err => { + this.logger.error(`get bucket obj names failed!`) + }) + if (!Array.isArray(minioObjNamesArr)) return; + minioObjNamesArr.forEach(item => { bloomFilter.add(item); }) + this.logger.debug(`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`) + } + @Get("/tile") async getTile( @Res({ passthrough: true }) response:Response, @Query('x') x:number, @Query('y') y:number, @Query('l') l:number, - @Query('type') type:string, + @Query('T') type:string, @Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e' ) { - this.logger.debug(`${x}_${y}_${l}_${type}_${tk}`) // 检查参数 if (!x || !y || !l || !type || !tk) return; const tileFormattedName = mapUrlFormatter(x, y, l, type); @@ -46,7 +60,6 @@ export class TdtmapController { // 获取到缓存中的内容， 直接返回 // @ts-ignore if (cacheItem && ArrayBuffer.isView(cacheItem)) { - this.logger.debug("item exist in cache!") Buffer.from(cacheItem as any, 'utf-8') return new StreamableFile(cacheItem as any); // 请求成功 } @@ -54,13 +67,11 @@ export class TdtmapController { // 缓存中不存在，查看本地数据库是否存在 const existInDB = bloomFilter.has(tileFormattedName); if (existInDB) { - this.logger.debug("item exist in db!") - // 本地数据库中存在，从本地数据库取出 - const dbItem:ArrayBufferView = this.tdtmapService.getTileFromLocal(tileFormattedName) // 从本地数据库取出来 + // minio 中存在，从 minio 取出 + const dbItem:Buffer = await this.tdtmapService.getTileFromLocal(tileFormattedName) // 从 minio 取出来 cacheMap.set(tileFormattedName, dbItem) - return response.send(dbItem); + return new StreamableFile(dbItem); } else { - this.logger.debug('start tdt request!') // 如果本地数据库都不存在 const res = await axios({ url: urlFormatter(x, y, l, type, tk), @@ -72,7 +83,9 @@ export class TdtmapController { // 写入缓存和本地 if (res && typeof res.data === 'object') { cacheMap.set(tileFormattedName, res.data) - this.tdtmapService.setTileToLocal(tileFormattedName, res.data); + const saveSuccess = await this.tdtmapService.setTileToLocal(tileFormattedName, res.data); + if (saveSuccess) bloomFilter.add(tileFormattedName); + Buffer.from(res.data, 'utf-8') return new StreamableFile(res.data); // 请求成功 } else { diff --git a/src/tdtmap/tdtmap.service.ts b/src/tdtmap/tdtmap.service.ts index dd1e327..697beb5 100644 --- a/src/tdtmap/tdtmap.service.ts +++ b/src/tdtmap/tdtmap.service.ts @@ -1,15 +1,14 @@ -import { Injectable } from '@nestjs/common'; -import { CreateTdtmapDto } from './dto/create-tdtmap.dto'; -import { UpdateTdtmapDto } from './dto/update-tdtmap.dto'; +import { Injectable, Logger } from '@nestjs/common'; import * as Minio from 'minio'; @Injectable() export class TdtmapService { private readonly minioClient: Minio.Client; + private readonly logger: Logger = new Logger(TdtmapService.name) constructor() { this.minioClient = new Minio.Client({ endPoint: '117.73.12.97', - port: 9001, + port: 9000, useSSL: false, accessKey: 'inspur', secretKey: 'inspur123', @@ -17,12 +16,40 @@ export class TdtmapService { } setMinioObj() {} + + async getTilesNameList():Promise { + return new Promise(async (resolve, reject) => { + const stream = await this.minioClient.listObjectsV2("nestfiles"); + const res:string[] = [] + stream.on("data", (obj) => { res.push(obj.name) }) + stream.on("end", () => { resolve(res) }) + stream.on("error", () => { reject() }) + }) + } - getTileFromLocal(tile_name:string):ArrayBufferView { - const buffer = new ArrayBuffer(16); // 创建一个ArrayBuffer - const view = new Uint8Array(buffer); // 创建一个视图 - return view; + async getTileFromLocal(tile_name:string):Promise { + return new Promise(async (resolve, reject) => { + const stream = await this.minioClient.getObject("nestfiles", tile_name) + let res:any = null + stream.on("data", (data) => { res = data }) + stream.on("end", () => { + if (res) resolve(res) + else reject() + }) + stream.on("error", () => { reject() }) + }) } - setTileToLocal(tile_name:string, data:ArrayBufferView) {} + + async setTileToLocal(tile_name:string, data:Buffer):Promise { + const res = await this.minioClient.putObject("nestfiles", tile_name, data).catch((err) => { + this.logger.error(`Minio Save Obj failed! => ${err}`) + }) + if (!res) return false; + return true; + } + + deleteMinioObj(tile_name:string) { + this.minioClient.removeObject("nestfiles", tile_name); + } }