From 53cf0fa2dd396cefc590b10c708deed872393362 Mon Sep 17 00:00:00 2001 From: pupuupup Date: Thu, 11 Mar 2021 09:55:19 +0700 Subject: [PATCH] fix grequest async working --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 0e0f216..888a200 100755 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ import re from lxml import html from tqdm import tqdm import codecs +import time def importData(): @@ -50,7 +51,7 @@ def getShare(datas): print("Getting Share ...") urls = list(map(getProfileUrl, datas)) rs = (grequests.get(u) for u in urls) - pages = tqdm(grequests.map(rs)) + pages = tqdm(grequests.imap(rs)) shares = list(map(scrapeAndFormatShare, pages)) for i in range(len(datas)): datas[i].update({'share': shares[i]}) @@ -70,7 +71,7 @@ def getPrice(datas): print("Getting Price ...") urls = list(map(getPriceUrl, datas)) rs = (grequests.get(u) for u in urls) - pages = tqdm(grequests.map(rs)) + pages = tqdm(grequests.imap(rs)) prices = list(map(scrapeAndFormatPrice, pages)) for i in range(len(datas)): datas[i].update({'price': prices[i]}) @@ -96,7 +97,7 @@ def getFinance(datas): print("Getting Finance ...") urls = map(getFinanceUrl, datas) rs = (grequests.get(u) for u in urls) - pages = tqdm(grequests.map(rs)) + pages = tqdm(grequests.imap(rs)) finances = list(map(scrapeAndFormatFinance, pages)) for i in range(len(datas)): datas[i].update(finances[i])