feat: 天地图Tile缓存

src/tdtmap/entities/tdtmap.entity.ts

@@ -1,7 +1,7 @@
 import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
 
 @Entity("tdt_local")
-export class Tdtmap {
+export class TdtLocal {
     @PrimaryGeneratedColumn()
     id: number;
     

src/tdtmap/entities/tdttile.entity.ts

@@ -1,19 +0,0 @@
-import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
-
-@Entity("tdt_tile")
-export class Tdtmap {
-    @PrimaryGeneratedColumn()
-    id: number;
-    
-    @Column({ type: 'varchar', length: 120, nullable: false })
-    url: string;
-
-    @Column({ type: 'varchar', length: 120, nullable: false })
-    name: string;
-
-    @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
-    create_time: Date;
-
-    @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
-    update_time: Date;
-}

src/tdtmap/tdtmap.controller.ts

@@ -13,7 +13,7 @@ const cacheMap = new LRUCache<string, ArrayBufferView>({
 })
 
 // 布隆过滤器
-const bloomFilter = new BloomFilter(10, 4)
+const bloomFilter = new Set<string>()
 
 const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`
 const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}`
@@ -23,20 +23,34 @@ const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${
 export class TdtmapController {
   constructor(
     private readonly tdtmapService: TdtmapService
-  ) {}
+  ) {
+    this.initBloomFilter()
+  }
   
   private readonly logger: Logger = new Logger(TdtmapController.name)
 
+  deleteAllMinioObj(tile_name:string) {
+    this.tdtmapService.deleteMinioObj(tile_name)
+  }
+  // 初始化过滤器
+  async initBloomFilter() {
+    const minioObjNamesArr = await this.tdtmapService.getTilesNameList().catch(err => {
+      this.logger.error(`get bucket obj names failed!`)
+    })
+    if (!Array.isArray(minioObjNamesArr)) return;
+    minioObjNamesArr.forEach(item => { bloomFilter.add(item); })
+    this.logger.debug(`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`)
+  }
+
   @Get("/tile")
   async getTile(
     @Res({ passthrough: true }) response:Response,
     @Query('x') x:number,
     @Query('y') y:number,
     @Query('l') l:number,
-    @Query('type') type:string,
+    @Query('T') type:string,
     @Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e'
   ) {
-    this.logger.debug(`${x}_${y}_${l}_${type}_${tk}`)
     // 检查参数
     if (!x || !y || !l || !type || !tk) return;
     const tileFormattedName = mapUrlFormatter(x, y, l, type);
@@ -46,7 +60,6 @@ export class TdtmapController {
     // 获取到缓存中的内容， 直接返回
     // @ts-ignore
     if (cacheItem && ArrayBuffer.isView(cacheItem)) {
-      this.logger.debug("item exist in cache!")
       Buffer.from(cacheItem as any, 'utf-8')
       return new StreamableFile(cacheItem as any); // 请求成功
     }
@@ -54,13 +67,11 @@ export class TdtmapController {
     // 缓存中不存在，查看本地数据库是否存在
     const existInDB = bloomFilter.has(tileFormattedName);
     if (existInDB) {
-      this.logger.debug("item exist in db!")
-      // 本地数据库中存在，从本地数据库取出
-      const dbItem:ArrayBufferView = this.tdtmapService.getTileFromLocal(tileFormattedName) // 从本地数据库取出来
+      // minio 中存在，从 minio 取出
+      const dbItem:Buffer = await this.tdtmapService.getTileFromLocal(tileFormattedName) // 从 minio 取出来
       cacheMap.set(tileFormattedName, dbItem)
-      return response.send(dbItem);
+      return new StreamableFile(dbItem);
     } else {
-      this.logger.debug('start tdt request!')
       // 如果本地数据库都不存在
       const res = await axios({
         url: urlFormatter(x, y, l, type, tk),
@@ -72,7 +83,9 @@ export class TdtmapController {
       // 写入缓存和本地
       if (res && typeof res.data === 'object') {
         cacheMap.set(tileFormattedName, res.data)
-        this.tdtmapService.setTileToLocal(tileFormattedName, res.data);
+        const saveSuccess = await this.tdtmapService.setTileToLocal(tileFormattedName, res.data);
+        if (saveSuccess) bloomFilter.add(tileFormattedName);
+        
         Buffer.from(res.data, 'utf-8')
         return new StreamableFile(res.data); // 请求成功
       } else {

src/tdtmap/tdtmap.service.ts

@@ -1,15 +1,14 @@
-import { Injectable } from '@nestjs/common';
-import { CreateTdtmapDto } from './dto/create-tdtmap.dto';
-import { UpdateTdtmapDto } from './dto/update-tdtmap.dto';
+import { Injectable, Logger } from '@nestjs/common';
 import * as Minio from 'minio';
 
 @Injectable()
 export class TdtmapService {
   private readonly minioClient: Minio.Client;
+  private readonly logger: Logger = new Logger(TdtmapService.name)
   constructor() {
     this.minioClient = new Minio.Client({
       endPoint: '117.73.12.97',
-      port: 9001, 
+      port: 9000, 
       useSSL: false,
       accessKey: 'inspur',
       secretKey: 'inspur123',
@@ -17,12 +16,40 @@ export class TdtmapService {
   }
 
   setMinioObj() {}
+
+  async getTilesNameList():Promise<string[]> {
+    return new Promise(async (resolve, reject) => {
+      const stream = await this.minioClient.listObjectsV2("nestfiles");
+      const res:string[] = []
+      stream.on("data", (obj) => { res.push(obj.name) })
+      stream.on("end", () => { resolve(res) })
+      stream.on("error", () => { reject() })
+    })
+  }
   
-  getTileFromLocal(tile_name:string):ArrayBufferView {
-    const buffer = new ArrayBuffer(16); // 创建一个ArrayBuffer
-    const view = new Uint8Array(buffer); // 创建一个视图
-    return view;
+  async getTileFromLocal(tile_name:string):Promise<Buffer> {
+    return new Promise(async (resolve, reject) => {
+      const stream = await this.minioClient.getObject("nestfiles", tile_name)
+      let res:any = null
+      stream.on("data", (data) => { res = data })
+      stream.on("end", () => {
+        if (res) resolve(res)
+        else reject()
+      })
+      stream.on("error", () => { reject() })
+    })
   }
 
-  setTileToLocal(tile_name:string, data:ArrayBufferView) {}
+
+  async setTileToLocal(tile_name:string, data:Buffer):Promise<boolean> {
+    const res = await this.minioClient.putObject("nestfiles", tile_name, data).catch((err) => {
+      this.logger.error(`Minio Save Obj failed! => ${err}`)
+    })
+    if (!res) return false;
+    return true;
+  }
+
+  deleteMinioObj(tile_name:string) {
+    this.minioClient.removeObject("nestfiles", tile_name);
+  }
 }