use get_keyOvalue

sync_amz_data/DataTransform/Data_ETL.py

@@ -89,6 +89,12 @@ class Common_ETLMethod(BaseClient):
         else:
             return '-'
 
+    def id_type_trans(self,df):
+        id_columns = [i for i in df.columns if 'id' in i.lower() and 'bid' not in i.lower()]
+        df[id_columns] = df[id_columns].fillna(-1).applymap(lambda x: round(int(x), 0)).astype("string")
+        df[id_columns] = df[id_columns].applymap(lambda x: '' if x == '-1' else x)
+        return df
+
     def type_trans(self, df_report, columns: list, timeZone_: str, extra_columns: list = []):
         # 添加字段
         df_report['profileId'] = self.profile_id
@@ -102,9 +108,7 @@ class Common_ETLMethod(BaseClient):
         columns.extend(["timeZone", "profileId"])
 
         # 修改字段类型
-        id_columns = [i for i in columns if 'id' in i.lower() and 'bid' not in i.lower()]
-        df_report[id_columns] = df_report[id_columns].fillna(-1).applymap(lambda x: round(int(x), 0)).astype("string")
-        df_report[id_columns] = df_report[id_columns].applymap(lambda x: '' if x == '-1' else x)
+        df_report = self.id_type_trans(df_report)
         df_report['date'] = df_report['date'].astype("datetime64")
 
         # df_report[df_report.select_dtypes('O').columns] = df_report[df_report.select_dtypes('O').columns].astype('string')
@@ -465,6 +469,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         # df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
         # df_targets = self.expression_split(df_targets, "resolvedExpressions")
+        df_targets = self.id_type_trans(df_targets)
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)
@@ -960,6 +965,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         list_targets = [row for _ in list(self.iter_targets(**param)) for row in _]
         df_targets = pd.json_normalize(list_targets)
         # df_targets = self.expression_split(df_targets, "resolvedExpression")
+        df_targets = self.id_type_trans(df_targets)
         df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
         df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)
@@ -1460,7 +1466,7 @@ if __name__ == '__main__':
     ac_etl = SB_ETL(**AWS_CREDENTIALS)
     ls = ac_etl.targets_ETL()
     print(ls)
-    print(ls.to_excel('obse111.xlsx'))
+    print(ls.to_excel('obse11.xlsx'))
     # ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
     # ac_etl.reportV2_campaignsRecord_t3_ETL(conn)
     # ac_etl.reportV2_adGroupsRecord_t2_ETL(conn)