h***@163.com
h***@163.com
  • 发布:2024-08-26 12:31
  • 更新:2024-08-27 11:47
  • 阅读:264

【报Bug】百万数据导出会出现接口频繁调用或者request:fail timeout

分类:uniCloud

产品分类: uniCloud/App

示例代码:

前端代码:


                var listQuery = {  
                    page : 1,  
                    total : 10000,  
                    isAdvs : false  
                }  
                listQuery.pageNum = Math.ceil((parseInt(this.listQuery.totalCount) / parseInt(listQuery.total)));  
                if(listQuery.pageNum <= 0 ) {  
                    this.$message({ message: '没有需要导出的数据', type: 'error' })  
                    return false;  
                }  
                listQuery.searchData = this.searchData;  
                this.exportProgress = 1;  

                var xlsxName = '交互数据'+convertTime((new Date()).getTime(),'yyyyMMddhhmm');  
                const workbook = new ExcelJS.Workbook();  
                // 创建一个带有页眉和页脚可写的工作表  
                var worksheetWriter = workbook.addWorksheet(xlsxName);  
                worksheetWriter.columns = [  
                    {header:'IP地址',key:'ip',width:32, style: {  
                            border:{  
                                top: {style:'thin',color: {argb:'a8abb2'}},  
                                left: {style:'thin',color: {argb:'a8abb2'}},  
                                bottom: {style:'thin',color: {argb:'a8abb2'}},  
                                right: {style:'thin',color: {argb:'a8abb2'}}  
                            }  
                        },  
                    },  
                    {header:'归属地',key:'ip_address',width:32, style: {  
                            border:{  
                                top: {style:'thin',color: {argb:'a8abb2'}},  
                                left: {style:'thin',color: {argb:'a8abb2'}},  
                                bottom: {style:'thin',color: {argb:'a8abb2'}},  
                                right: {style:'thin',color: {argb:'a8abb2'}}  
                            }  
                        },  
                    },  
                    {header:'时间',key:'create_time',width:32, style: {  
                            border:{  
                                top: {style:'thin',color: {argb:'a8abb2'}},  
                                left: {style:'thin',color: {argb:'a8abb2'}},  
                                bottom: {style:'thin',color: {argb:'a8abb2'}},  
                                right: {style:'thin',color: {argb:'a8abb2'}}  
                            }  
                        },  
                    }  
                ];  
                for(var i=0; i<worksheetWriter.columns.length;i++) {  
                    var cell = worksheetWriter.getRow(1).getCell(i+1)  
                    cell.fill = {   
                        type: 'pattern',   
                        pattern: 'solid',   
                        fgColor: {argb: 'E9E9EB'}  
                    };  
                    cell.font = {  
                        family: 4,  
                        size: 16,  
                        bold: true,  
                        vertAlign: 'middle',  
                    };  
                    cell.alignment={  
                        vertical: 'middle',   
                        horizontal: 'center',  
                        wrapText: true   
                    };  

                }  

                let data = await this.exportList(listQuery);  
                while(data.length > 0) {  
                    for(var i in data) {  
                        data[i].create_time = convertTime(data[i].create_time);  
                        worksheetWriter.addRow(data[i])  
                    }  
                    data =  await this.exportList(listQuery);  
                }  
                worksheetWriter.eachRow((row,inx) => {  
                    if(inx > 1) {  
                        row.height = 20;  
                    }else {  
                        //第一行设置高度32  
                        row.height = 32;  
                    }  
                })  
                workbook.xlsx.writeBuffer().then(buffer => {  
                    saveAs(new Blob([buffer],{type:'application/octet-stream'}),xlsxName + '.xlsx')  
                })  

            },  
            async exportList(listQuery){  
                return new Promise((resolve) => {  
                    setTimeout(async () => {  
                        if(listQuery.pageNum <= listQuery.page) {  
                            this.exportProgress = 0;  
                            resolve([]);  
                            return true;  
                        }  
                        let res = await this.commandHandleAjaxCloud(this.importObjectName,this.listApi,listQuery);  
                        if(!res) {  
                            resolve([]);  
                            return true;  
                        }  
                        listQuery.page += 1;  
                        this.exportProgress = Math.floor(((100/listQuery.pageNum) * (listQuery.page)));  
                        this.exportProgress = (this.exportProgress >= 100) ? 0 : this.exportProgress;  
                        resolve(res.data.list);  
                        return true;  
                    },100);  
                });  
            }```   

后端代码:
``` ['cms/advs/interact']: async function(event){  
        Validator(event,'list',validata);  
        var map = {};  
        map = initSearchData(event.searchData,map);  
        if(map.ip_address) {  
            map.ip_address = new RegExp(`${map.ip_address}`);  
        }  
        var total = !event.total ? 10 : event.total;  
        var page = !event.page ? 1 : event.page;  
        // var offset = total * (page - 1);  
        var order = (!event.order || event.order.length === 0) ? ['create_time','desc'] :event.order;  
        var n = Math.ceil(total / 1000);  
        var i = 0;  
        let res = [];  

        do{  
            i++;  
            var p = parseInt(n * (page - 1)) + i;  
            var offset = Math.floor(total / n) * (p - 1);  
            console.log(n,i,p,page,offset)  
            res = await db.collection('advs_hit').where(map).orderBy(...order).skip(offset).limit(total).get();  
        }while(i < n)  
        let totalCount = await db.collection('advs_hit').where(map).count();  
        let advs = [];  
        if(event.isAdvs) {  
            advs = await db.collection('advs').where({status:1}).limit(1000).get();  
        }  
        return success({  
            list : res.data||[],  
            totalCount : totalCount.total,  
            advs : advs.data||[],  
        });  
    }```

操作步骤:

我需要导出这百万数据,请求取数据,会超时或者接口频繁调用拦截

预期结果:

我需要导出这百万数据

实际结果:

请求取数据,会超时或者接口频繁调用拦截

bug描述:

方式一:我按每次请求出1万数据,会报错Request was denied due to api flow control
方式二:我按每次请求出5万数据,会报错request:fail timeout

这是页面
前端代码
后端代码

2024-08-26 12:31 负责人:无 分享
已邀请:
DCloud_uniCloud_VK

DCloud_uniCloud_VK

目前单次查询最大是1000条数据,因此深度分页时, 切记不能依赖skip, skip这个功能是跳页, 如果值很大, 那么性能越差, 因此skip的值比如在100万时, 性能已经非常差了, 所以我们不能靠skip来遍历全表

那么如何正确的遍历全表呢?

首先建议放弃并发查库,然后再合并数据的想法(理由如上)

建议使用游标分页法, 即通过_id从小到大排序,再通过 where _id > 本页最后一条记录的_id 来获取下一页的数据, 此时下一页不需要通过skip, 即skip始终是0, 这种情况下, 性能不会下降, 只要你的云函数内存够, 超时时间够长,就可以遍历出全表数据

要回复问题请先登录注册