Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/public/amz_ad_client.py

@@ -320,8 +320,11 @@ class SBClient(BaseClient):
         }
         return self._request(url_path, method="POST", body=body, headers=headers)
 
-    def get_campaign_v3(self,campaignId):
-        url_path = f'/sb/campaigns/{campaignId}'
+    def get_campaign_v3(self, campaignId):
+        if campaignId is None:
+            url_path = f'/sb/campaigns'
+        else:
+            url_path = f'/sb/campaigns/{campaignId}'
         return self._request(url_path, method="GET")
 
     def iter_campaigns(self, **body) -> Iterator[dict]:

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign_v3.py → sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign.py

@@ -59,11 +59,16 @@ class SbCampaign:
     def get_sbcampaign_data(self):
         tem = SBClient(**self.AWS_CREDENTIALS)
         if self.portfolioId is None:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True})
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
+                                                   "stateFilter": {
+                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                                   }})
         else:
             list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
-                                                   "portfolioIdFilter": {"include": self.portfolioId}
-                                                   })
+                                                   "portfolioIdFilter": {"include": self.portfolioId},
+                                                   "stateFilter": {
+                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                                   }})
         df_campaigns = pd.json_normalize(list(list_campaigns))
         return df_campaigns
 

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign_v4.py

@@ -1,118 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SBClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SbCampaign:
-    def __init__(self, profile_id, portfolioId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = portfolioId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/sbcampaigns/list/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_sbcampaign_data(self):
-        tem = SBClient(**self.AWS_CREDENTIALS)
-        if self.portfolioId is None:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True})
-        else:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
-                                                   "portfolioIdFilter": {"include": self.portfolioId}
-                                                   })
-        df_campaigns = pd.json_normalize(list(list_campaigns))
-        return df_campaigns
-
-    def dataconvert(self):
-        df = self.get_sbcampaign_data()
-        df['profile_id'] = self.profile_id
-        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'], unit='ms').dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'], unit='ms').dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['budgetType', 'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
-               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId', 'brandEntityId',
-               'isMultiAdGroupsEnabled', 'goal', 'bidding.bidOptimization', 'bidding.bidAdjustmentsByShopperSegment',
-               'bidding.bidAdjustmentsByPlacement', 'bidding.bidOptimizationStrategy', 'endDate',
-               'campaignId', 'productLocation', 'portfolioId', 'costType', 'smartDefault', 'name', 'state',
-               'startDate', 'budget', 'extendedData.servingStatus', 'extendedData.lastUpdateDate',
-               'extendedData.creationDate', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'campaignName',
-            'portfolioId': 'portfolio',
-            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
-            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
-            'ruleBasedBudget_value': 'rbb_value',
-            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId',
-            'bidding_bidOptimization': 'bidOptimization',
-            'bidding_bidAdjustmentsByShopperSegment': 'bidShopperSegment',
-            'bidding_bidAdjustmentsByPlacement': 'bidPlacement',
-            'bidding_bidOptimizationStrategy': 'bidOptimizationStrategy',
-            'extendedData_lastUpdateDate': 'lastUpdateDate',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails',
-            'extendedData_creationDate': 'creationDate'}, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SbCampaign(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -58,7 +58,10 @@ class SbGroup:
 
     def get_spgroup_data(self):
         tem = SBClient(**self.AWS_CREDENTIALS)
-        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True})
+        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True,
+                                          "stateFilter": {
+                                              "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                          }})
         df_group = pd.json_normalize(list(list_group))
         return df_group
     #----------------------------