feat: 天地图Tile缓存
This commit is contained in:
parent
13aea532df
commit
7c117d0440
@ -1,7 +1,7 @@
|
|||||||
import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
|
import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
|
||||||
|
|
||||||
@Entity("tdt_local")
|
@Entity("tdt_local")
|
||||||
export class Tdtmap {
|
export class TdtLocal {
|
||||||
@PrimaryGeneratedColumn()
|
@PrimaryGeneratedColumn()
|
||||||
id: number;
|
id: number;
|
||||||
|
|
||||||
|
@ -1,19 +0,0 @@
|
|||||||
import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
|
|
||||||
|
|
||||||
@Entity("tdt_tile")
|
|
||||||
export class Tdtmap {
|
|
||||||
@PrimaryGeneratedColumn()
|
|
||||||
id: number;
|
|
||||||
|
|
||||||
@Column({ type: 'varchar', length: 120, nullable: false })
|
|
||||||
url: string;
|
|
||||||
|
|
||||||
@Column({ type: 'varchar', length: 120, nullable: false })
|
|
||||||
name: string;
|
|
||||||
|
|
||||||
@Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
|
|
||||||
create_time: Date;
|
|
||||||
|
|
||||||
@Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
|
|
||||||
update_time: Date;
|
|
||||||
}
|
|
@ -13,7 +13,7 @@ const cacheMap = new LRUCache<string, ArrayBufferView>({
|
|||||||
})
|
})
|
||||||
|
|
||||||
// 布隆过滤器
|
// 布隆过滤器
|
||||||
const bloomFilter = new BloomFilter(10, 4)
|
const bloomFilter = new Set<string>()
|
||||||
|
|
||||||
const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`
|
const urlFormatter = (x: number, y: number, z: number, type: string = 'vec_w', token: string) => `http://t0.tianditu.gov.cn/DataServer?T=${type}&x=${x}&y=${y}&l=${z}&tk=${token}`
|
||||||
const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}`
|
const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${x}&Y_${y}&Z_${z}&type_${type}`
|
||||||
@ -23,20 +23,34 @@ const mapUrlFormatter = (x: number, y: number, z: number, type: string) => `X_${
|
|||||||
export class TdtmapController {
|
export class TdtmapController {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly tdtmapService: TdtmapService
|
private readonly tdtmapService: TdtmapService
|
||||||
) {}
|
) {
|
||||||
|
this.initBloomFilter()
|
||||||
|
}
|
||||||
|
|
||||||
private readonly logger: Logger = new Logger(TdtmapController.name)
|
private readonly logger: Logger = new Logger(TdtmapController.name)
|
||||||
|
|
||||||
|
deleteAllMinioObj(tile_name:string) {
|
||||||
|
this.tdtmapService.deleteMinioObj(tile_name)
|
||||||
|
}
|
||||||
|
// 初始化过滤器
|
||||||
|
async initBloomFilter() {
|
||||||
|
const minioObjNamesArr = await this.tdtmapService.getTilesNameList().catch(err => {
|
||||||
|
this.logger.error(`get bucket obj names failed!`)
|
||||||
|
})
|
||||||
|
if (!Array.isArray(minioObjNamesArr)) return;
|
||||||
|
minioObjNamesArr.forEach(item => { bloomFilter.add(item); })
|
||||||
|
this.logger.debug(`bloomFilter inited => minio obj num: ${minioObjNamesArr.length}`)
|
||||||
|
}
|
||||||
|
|
||||||
@Get("/tile")
|
@Get("/tile")
|
||||||
async getTile(
|
async getTile(
|
||||||
@Res({ passthrough: true }) response:Response,
|
@Res({ passthrough: true }) response:Response,
|
||||||
@Query('x') x:number,
|
@Query('x') x:number,
|
||||||
@Query('y') y:number,
|
@Query('y') y:number,
|
||||||
@Query('l') l:number,
|
@Query('l') l:number,
|
||||||
@Query('type') type:string,
|
@Query('T') type:string,
|
||||||
@Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e'
|
@Query('tk') tk:string = '6988fa4ec7ca5ed400097b9bf9dfc22e'
|
||||||
) {
|
) {
|
||||||
this.logger.debug(`${x}_${y}_${l}_${type}_${tk}`)
|
|
||||||
// 检查参数
|
// 检查参数
|
||||||
if (!x || !y || !l || !type || !tk) return;
|
if (!x || !y || !l || !type || !tk) return;
|
||||||
const tileFormattedName = mapUrlFormatter(x, y, l, type);
|
const tileFormattedName = mapUrlFormatter(x, y, l, type);
|
||||||
@ -46,7 +60,6 @@ export class TdtmapController {
|
|||||||
// 获取到缓存中的内容, 直接返回
|
// 获取到缓存中的内容, 直接返回
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
if (cacheItem && ArrayBuffer.isView(cacheItem)) {
|
if (cacheItem && ArrayBuffer.isView(cacheItem)) {
|
||||||
this.logger.debug("item exist in cache!")
|
|
||||||
Buffer.from(cacheItem as any, 'utf-8')
|
Buffer.from(cacheItem as any, 'utf-8')
|
||||||
return new StreamableFile(cacheItem as any); // 请求成功
|
return new StreamableFile(cacheItem as any); // 请求成功
|
||||||
}
|
}
|
||||||
@ -54,13 +67,11 @@ export class TdtmapController {
|
|||||||
// 缓存中不存在,查看本地数据库是否存在
|
// 缓存中不存在,查看本地数据库是否存在
|
||||||
const existInDB = bloomFilter.has(tileFormattedName);
|
const existInDB = bloomFilter.has(tileFormattedName);
|
||||||
if (existInDB) {
|
if (existInDB) {
|
||||||
this.logger.debug("item exist in db!")
|
// minio 中存在,从 minio 取出
|
||||||
// 本地数据库中存在,从本地数据库取出
|
const dbItem:Buffer = await this.tdtmapService.getTileFromLocal(tileFormattedName) // 从 minio 取出来
|
||||||
const dbItem:ArrayBufferView = this.tdtmapService.getTileFromLocal(tileFormattedName) // 从本地数据库取出来
|
|
||||||
cacheMap.set(tileFormattedName, dbItem)
|
cacheMap.set(tileFormattedName, dbItem)
|
||||||
return response.send(dbItem);
|
return new StreamableFile(dbItem);
|
||||||
} else {
|
} else {
|
||||||
this.logger.debug('start tdt request!')
|
|
||||||
// 如果本地数据库都不存在
|
// 如果本地数据库都不存在
|
||||||
const res = await axios({
|
const res = await axios({
|
||||||
url: urlFormatter(x, y, l, type, tk),
|
url: urlFormatter(x, y, l, type, tk),
|
||||||
@ -72,7 +83,9 @@ export class TdtmapController {
|
|||||||
// 写入缓存和本地
|
// 写入缓存和本地
|
||||||
if (res && typeof res.data === 'object') {
|
if (res && typeof res.data === 'object') {
|
||||||
cacheMap.set(tileFormattedName, res.data)
|
cacheMap.set(tileFormattedName, res.data)
|
||||||
this.tdtmapService.setTileToLocal(tileFormattedName, res.data);
|
const saveSuccess = await this.tdtmapService.setTileToLocal(tileFormattedName, res.data);
|
||||||
|
if (saveSuccess) bloomFilter.add(tileFormattedName);
|
||||||
|
|
||||||
Buffer.from(res.data, 'utf-8')
|
Buffer.from(res.data, 'utf-8')
|
||||||
return new StreamableFile(res.data); // 请求成功
|
return new StreamableFile(res.data); // 请求成功
|
||||||
} else {
|
} else {
|
||||||
|
@ -1,15 +1,14 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
import { CreateTdtmapDto } from './dto/create-tdtmap.dto';
|
|
||||||
import { UpdateTdtmapDto } from './dto/update-tdtmap.dto';
|
|
||||||
import * as Minio from 'minio';
|
import * as Minio from 'minio';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class TdtmapService {
|
export class TdtmapService {
|
||||||
private readonly minioClient: Minio.Client;
|
private readonly minioClient: Minio.Client;
|
||||||
|
private readonly logger: Logger = new Logger(TdtmapService.name)
|
||||||
constructor() {
|
constructor() {
|
||||||
this.minioClient = new Minio.Client({
|
this.minioClient = new Minio.Client({
|
||||||
endPoint: '117.73.12.97',
|
endPoint: '117.73.12.97',
|
||||||
port: 9001,
|
port: 9000,
|
||||||
useSSL: false,
|
useSSL: false,
|
||||||
accessKey: 'inspur',
|
accessKey: 'inspur',
|
||||||
secretKey: 'inspur123',
|
secretKey: 'inspur123',
|
||||||
@ -18,11 +17,39 @@ export class TdtmapService {
|
|||||||
|
|
||||||
setMinioObj() {}
|
setMinioObj() {}
|
||||||
|
|
||||||
getTileFromLocal(tile_name:string):ArrayBufferView {
|
async getTilesNameList():Promise<string[]> {
|
||||||
const buffer = new ArrayBuffer(16); // 创建一个ArrayBuffer
|
return new Promise(async (resolve, reject) => {
|
||||||
const view = new Uint8Array(buffer); // 创建一个视图
|
const stream = await this.minioClient.listObjectsV2("nestfiles");
|
||||||
return view;
|
const res:string[] = []
|
||||||
|
stream.on("data", (obj) => { res.push(obj.name) })
|
||||||
|
stream.on("end", () => { resolve(res) })
|
||||||
|
stream.on("error", () => { reject() })
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
setTileToLocal(tile_name:string, data:ArrayBufferView) {}
|
async getTileFromLocal(tile_name:string):Promise<Buffer> {
|
||||||
|
return new Promise(async (resolve, reject) => {
|
||||||
|
const stream = await this.minioClient.getObject("nestfiles", tile_name)
|
||||||
|
let res:any = null
|
||||||
|
stream.on("data", (data) => { res = data })
|
||||||
|
stream.on("end", () => {
|
||||||
|
if (res) resolve(res)
|
||||||
|
else reject()
|
||||||
|
})
|
||||||
|
stream.on("error", () => { reject() })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async setTileToLocal(tile_name:string, data:Buffer):Promise<boolean> {
|
||||||
|
const res = await this.minioClient.putObject("nestfiles", tile_name, data).catch((err) => {
|
||||||
|
this.logger.error(`Minio Save Obj failed! => ${err}`)
|
||||||
|
})
|
||||||
|
if (!res) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
deleteMinioObj(tile_name:string) {
|
||||||
|
this.minioClient.removeObject("nestfiles", tile_name);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user