🎈 perf: 同步数据分批请求

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spads.py

@@ -5,6 +5,7 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 from sync_amz_data.tasks.datainsert.wg import LADS
+import time
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
@@ -86,11 +87,17 @@ class SpAds:
         return json_data
 
     def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
+        my_list = self.dataconvert()
+        chunk_size = 500
+        chunked_lists = [my_list[i:i + chunk_size] for i in range(0, len(my_list), chunk_size)]
+        temlist = []
+        for body in chunked_lists:
+            heads = self.heads
+            url_path = self.upcreate_url_path
+            tem = request(url_path=url_path, head=heads, body=body, method="POST")
+            temlist.append(tem)
+            time.sleep(5)
+        return temlist
 
 
 if __name__ == '__main__':

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -5,6 +5,8 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 from sync_amz_data.tasks.datainsert.wg import LADS
+import time
+
 
 class RateLimitError(Exception):
     def __init__(self, retry_after: str = None):
@@ -88,11 +90,17 @@ class SpKeyword:
         return json_data
 
     def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
+        my_list = self.dataconvert()
+        chunk_size = 500
+        chunked_lists = [my_list[i:i + chunk_size] for i in range(0, len(my_list), chunk_size)]
+        temlist = []
+        for body in chunked_lists:
+            heads = self.heads
+            url_path = self.upcreate_url_path
+            tem = request(url_path=url_path, head=heads, body=body, method="POST")
+            temlist.append(tem)
+            time.sleep(5)
+        return temlist
 
 
 if __name__ == '__main__':

sync_amz_data/tasks/datainsert/mysql_datainsert_assets.py

@@ -5,6 +5,7 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 from sync_amz_data.tasks.datainsert.wg import LADS
+import time
 
 
 class Client(BaseClient):
@@ -96,11 +97,17 @@ class Assets:
         return json_data
 
     def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
+        my_list = self.dataconvert()
+        chunk_size = 500
+        chunked_lists = [my_list[i:i + chunk_size] for i in range(0, len(my_list), chunk_size)]
+        temlist = []
+        for body in chunked_lists:
+            heads = self.heads
+            url_path = self.upcreate_url_path
+            tem = request(url_path=url_path, head=heads, body=body, method="POST")
+            temlist.append(tem)
+            time.sleep(5)
+        return temlist
 
 
 if __name__ == '__main__':

sync_amz_data/tasks/datainsert/wg.py

@@ -1,2 +1,2 @@
-LADS = "http://192.168.1.19:8001/"
-# LADS = "https://ads.vzzon.com"
+# LADS = "http://192.168.1.36:8001/"
+LADS = "https://ads.vzzon.com"