diff --git a/CKAN_JRO-master.tar.gz b/CKAN_JRO-master.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..77750a6bdb27b0a6e9162f6dc719ea480e1f6cca GIT binary patch literal 6611 zc$|fpRa6uXwDo}jhVBLd=|%w=N|25rq{9(uknR+Q?(UEV>Fy9jdT5ZYp+i8rnYsUm z?|Z!Ko~Lux+Uxgp_I}t5F<4mKc5`NENLjG7`WF=~4PFORH+M@{K6g`Bl+*Xiu4^L6 z0kw6%Qf2zgz(vQ?GqnyrMs{hd$F1>6yGKVe98@~wDrPzIvl2WN;$mh{^Imf)6J}4O zMX!Y*Dm~n=S~hu|?YAkpG-Hy|ZWuZiM-FWNu59JyQ)OKF)wZTqPFYd!sb?J}a_Cvz z+OF#=c&*nvc(*HW|HIYT*)C;rXP3qH5ozb*@-m`z{q(H+TZR3Nm^gn|9HqzORoSA{ zDMOW1Uaq~W&|>lAzWFHf>`WMOdg9nN*n1kwXL7l@y0PIa-I}k3P&<}LP0P7O;HE)K z7mzK8x*KQ>67a$x>BMJ()y_J^Zx*KF& ze5q|~?Gj-^HPY+9nSF$|tz$emNtHfBOCO;p2BZSd1zB$vPa6<# za{qDR8OD!!^=YH$-?~veg7#I-&JOJuXlrrvoh-xGJjX0Rb-nsz#Xg=Bavy+AD%uBV zZHD?%{dASoJea04BBc$4(5C*hL9gi~biAJv7ILx|O*vnV-tcxeOYjI$NOSbp0R1l8 zydD@Uv?TB!F8omz>p$gXgFaNq;D78pb5&Ps8Qv=mp@|il){dqTVgcWeZ~8JQrkvv= z_B(0=7_2(;igQ4fcP6r3afO8)e5elq-{#$bkPo)PIVb;e`gdNuB7jdo3EH`b=@|#c z(0v;@S~YiVpmVk?eWb^!rir1MO(fB?BK~2#Lq{hNU`1?`qk4!H5XTWT1S#z+$iqk6 zBF>-Bx6gOKfB)|wfbQ%Ik5ud%mCUhI;(5<4IJo*eqNtiApXsI;@KjOm-7so^q<`Z% zSXjc{pLP>_(It2pVR3c+93QEYqR6j@2MFSL*lT0Ink3iH%Si&`4KDQJ5<6?N8b16BQMaka{mF`F!UUGJs_(OQ!m| zzYQNuTcOG%&_TL+?!H=Z(!$;C!>Gsfj znr9IaF@N^PEfw}re6LO~T3D?6Ums7Xn;W{AYT?}qdZK<{eu)PO7x#~>+mf_5$^uw| z59+i86@M##2p85pQWtSY%Rq2T)*37N8Ji*0ZsygUqej?)7W{E0x@$B|rb%B7rSr7I z!}n&>>0tLqqsxN`>SgF%|C2e<6{$&b@e9=pz*i0XozxLyW)Jo@)(SCt@-$Ez^lcSs z+}n^w``4&}GnU!4&?=rM(G)~sMyasYvPNa2$QHruU7X-yapHa0B_MPd%+|H=3e6Vg z$@B4%l$UEdWfb`gQli#jHs7R^X~R|uXQM#a>GSAE*R@jb}4teL^$0yO3n~#;auE~E1aY<*QtSoj{vQskxkS_$oF0)y{ zZ@bQaD=s*RB|4FIJI;#DhIbwro}KmvnO&oRMi>hskst3V|QMfn>Si#zgjj9;^D z7LCBvXG0);w_9lmnZO92JT7)PAt#~^I_O^(4lYaEOr_y8T7TRJ&STU8jlEXZYj*7b z_HVk|yH%_bu3uXr5sCt|-T+!{M{5N2{%!v9e?zkA0#(CtW%0&_8BDSc!xRLP7M~76Bt_SOuOCno=}z*0 z27tA~^yGRD{;+j|RdLsy1P8l(MbuU4?eW7e!zz=kR5aJ{e9_!VU`{1Na0Ka=EO zI@$cg-#Z3nLrR@66o3y)HTpg4+go-rzLA&`MrSw56Zxd-jf zAc?*})1&CJ7Hs>Cze=2*ZW$8mFS?6DS}UP)!LN={{X=E?yMlWq z4Q1ci(g@bq&5|2N`OXl9(N=p|=f0^u9tf^FGznZ!|E?)lURVz!^B~F^0`tUfkAE5= zt7fUic8H=;_Jdty`YZT;zVQDlGw>=LzYrvsK}p1^rdFNW;3J!7UdXEv(%W zm#w^fC}3$b^IRLVVX)|YFa6+bwH`7WzfAHcWfuY#I4>IczY$-ME*I##yjbn@li@9VM1z!3gT!&528F4?Ow9XHjP zYjHFf@3@J|91T!fl#+XBH^0G5LQhcdyT(zIKrM~Bpp5)xZ#Ryv`15l}^$7!AB8c5u z@rg1F2a|4&6b%q2WXIHak`Y}lJm$f7+d-aUr10fk)qPSi&s-*2*yG`L#)@*DKHpB%Y`0ebQMZIgY}bs{)zzV1~J5 z&94g#sz@--hspZzgkGc05i=80i-*dRpG*Y(-m*FzP&^`KI(Q@@7^b&V`&->a@RF%| z7)7Y^N$dG}lHgVsimP&=73uoukcEsKlchv%V3Z}iET2%ORBn_UAqLDo; zL)RNMOO1P>Gq`D;lIM|unw2^y4UIL6s(*yt%-9^#7AmQ{%N00MKt)^6#lcoax}*Ss zVhVx#cqTRW9-71S5pl&8X!Lo^-!!W>04_> zyi^j_8reS5+sI|x(V#n$(sYrrX?h8m$8}L1v>)oKXiSJEaUXe>E<;*|q#Y&}JZT0b zDDoG{Ti9uF)p7n59scAS`UJnLP}b0mOxc-!i+T1VRL8V~xs5oi5gdiI($# zv6R{Ikz3&Xz#2 zK;^!9zKJo~j|B!ep2qiP`$P(ky~W32xu)VC!Rl~d?7x&NXUCt3C~6IM#@ttmMDWgx zv3~Q7|E#x{450Eed{c`?^GD#FEK-g3s|0L2ho<9pJwY;z&bHCxs8DhtV&Q=FqbWME zQWAc_E>L$rN{8hcya7S{d8;pq(ZVdzw~F;_U*V3a&D1=jLQRCqqQ`K`de1n_JtobR z`rOeFsWU8NvlM^Bty;4`^xM@@NYKFT3tmd0CDlsHpbRZ8VH{R02CL3AJC~zeCT%Zv zE=1c}G}MSd;mzGlt-q`r{r(UaX2TNPXC+RDO3AzlP{^8jqGt%hw;xD4>l_zmVHD~h zKO_CfEE%q4u%ygv9ue$?u{7eQg?`MZfFoRT6%SnHYWjH9YqaRrM&My>(`}0t8D<}> zhGipXpAetf#Laa*?4sL-0T=^!iKOE*llKv!{*P((bemM)fcw)Y>cMiBjoZP@6@5Ln8a#+&Z zkO{6tF=5TPj|@05(F|kW60xx-zmx?0-ATy0#{r&c{A2Q?RkOC4zX=7L>q_IKVCN_s zkQYmHkH!YEzSi#n z(B}yZBvhlC)Tgr|iOa3i5L7Xiu%4whOQis;I=M6JK5yx;JZse}*3fRIu+*%YT-Qc9 z64!pTI)V$wKL5#-Qfe$+W!Z3!-NWJbsa{ry--V-8)xAPMw!8kBdTzmTWKX}80RNc5)0%S^Aceg>OqFWfDp&{Ew+)PO@ClctOb%f@*;!Y~o6M2~B3@#dQD`F3~2! zhy$hoZd9>Gg?RSzB=|adr2kVbvtYa})=o9}6{VhB_$x#mHab^~;_qcwEF}jNT4klv`|myB8a@(Wf{Y&b z#VM}Rdop^~;?zX*r_4O3uVx2Nq?YwSrhdm)4*WTkQI=1_p2W1lyDY>%=$`KeSsP8Zx6H{8d3%QTW5LUQ zK;^dwE?oxTPNYLB@U6geKR0G|NH^ihoE#NK!{Vi~$m((52-oWkI`2|ZKa6EUl#D1p z8DixTXKcBV;LOlZXd-aiNS!!C)p^-&Lh##QsB5^gLEXVYc8To8a?|AHw+x&TXt3yB zw*ik>+4r{f!GqK^quQ6Ex70fd@27^HI?V4zE``irp>+0aU4(R_LnHIaT@yC->r>p$ zZ?aiFXu??oT%6f?zUkDKD9ea|s!P|l4n%m5;JXJqUhGf65WxAs2ENfjW4&q=c#&@J ze$KChD7ATrP$c+>zh!R#^m5`}#ZtmjI#++r;}Lj}f|j|bMw#x{MCi|R%t>2ZUQ7!< z`c{vSdgVU3WlFq7(&F}BoUFJ)C@*EfuweuZOVxuSkdZ7>ixK}`8-ICC%FID564qjOO2S*ecsnX9QIVC;*BT&^{Zn;8$!1q|9{hy@ za@7j?6ZpIxp48#KT7I2kuP=Fxtm5ey^}E8xrqayXdy(wsb6N1|3j83VGw5bnFc_fp zS=2gF)wgW>!#{aC_(uSeEx`p>Y%DuvQF%WS^vtC`Dn5^|Om!F|5Qs`Mm|Dop66IX( z=e{#^16cZO3j_e8r(0Ds6tY?0yJsm2bOkbZ*U6zem6QK&PyIHGMIcnP!m`xlH^!Ia zb{CLAV*Lz$`5uFUFllpR?1uF9Q>vJBL@Y!WtsJ-F3WvXy$Qq>dYo%Zw^N{kzU3}If zDsgC-N_g&f1Yn&+#QAB^k%xt&Et67Wl++R=mYr#vSeEhcF^8ji)zcig+3&b)O?}YP z7v1p*dBZ*vThrmF)tI8+6xs&l@zbTM<0?8MGTBbJe`EAR;~|YH@ib?v!#7_O&ZV5c z_bM&8i$(|VLtWug;zyc~SNhgBnrr2)$>KL3DIC}O8o$xv#_$-VYi`lKE&mgHr-S+8 zV79kk#Z@Lh^pk*yXcA;T#Y&)Oybj8K4FNR~ora3aSkgp$Sp6vd=<`)bjge(qIO$8$ z&{qR+$vdM?n4DD@SQug%U#Of_-4koWOI8Q@v+?JR~1!G z^YhFeH;~V-Kz)jqSLfXga&>H-m?Ktgbrt!)O5pyymzFsR)^%U%X`9?L{KQ1yTXfvS zk{sX3!x#6-$k7Sh4+kFQh;8~LRUMl(Dg^sp<@b?C&-eE|jq?Q+3VC#=Ui4k|C2G<0 zMDy7V@aBwL!sFZCWK9vQ@Dx@~<<(9p4`0Sb3ev%H&4C}1!fMesA)w7Mi~V@c%h!SN2m|N!wNxB=oEl?Uf9M7{4Ut-da0N>GC<6 z)=G(CY`QyZL`YtD46@A$c^^}L*-4D;!%B+ez#}dGUN>(3N3e}gj(7(Rx357Pg7hUZ zo=hZ2swRcLr@5ia4s+@Jv)pmfMHI397oX{Hjt_)|yAN?7sz;@Ltj9H}mEj{I%us8R zqnH8&EudU`dEM+fxliTYC&c?;z*Pz$B~1wy{1dZRF4CB3#AZjFfitzQLrFIs4@DtsShz|QY_Yj!=peHfad{;iq|EQdH0@nNC?%$ znY8UvV?CgsPk&6i=r!8-CrH9yy1YtY2 zGO^%SGiiWU7?_X4_yj-uicIXyDPQ)kQJ*D)Qy{0O*2BB~Xu%6Qm6S9Z!GPfQ*-T9u z8GW64t>K6RCm>L_h;ZIUJC{8_G`KY7PUq@1|HMK-qu%S$1dVp^hXDHT_)6U;Jh}$Z z1M&n7zLA`iyJ6sLjERP8jwhDFz zjv^-)&K7VRUs5lfv*kWx{Ie^xipF1?o)M8Y-lNJGk= zj=opaP*up&vlZtCT9=WITtEGsh`iN?A@wVf3FT#50nu^2SMyDhM}f}_{Us$DiIl>D zlr%HqlPqy=P=T565t5>T*}FX5ocBTYL_COv>}mW@U8qF=5};z?4=N!}M@K1rBQ2_z zF%`5urzy-6S3Dr^4r%sm$7^hVxKr<*#>YXX^NKk}bl^MZbN-Uw?*EkX`Y7dUwcf%k#_8qw*B$fc%eKR=R|gnD_~K zB23_X+Fai_UbsbaYVjVoywv?a#VYdKQ}2DMedWu)l2d5e>T<;?Qac{Xd-Lk5vNpb?fgj?X`tdS1k58!jI%nP!s+JZk{UC;MhJPGjDt2ky80D8iUBT-j~*R zB=qKXGr*wrIl~{>GOzrETwTxgMQ)~)kjqRIzB99?z2ETOSshwxU<+*svNj9{yFXj3 z?uf==KF!&NMi>S9P2(AscT6^hn75zfA5<_x$uyQPK06*Va=O zQD0-^I1;b^a89;nytR!@_x4O5ze5kxqf$HfSf@;EzC!)~SdU^: + ... some operation(s) ... + #2: + = JROAPI('http://example.com', Authorization='#########') + ... some operation(s) ... + .ckan.close() + + REPORTAR ALGUN PROBLEMA: + Debe enviar un correo a eynilupu@igp.gob.pe detallando los siguientes pasos: + 1) Identifiquese + 2) Describir el problema + 3) ¿En que funcion esta el problema? + 4) ¿Que esperaba que hiciera la funcion sin el problema? + """ + def __init__(self, url, Authorization=None): + ua = 'CKAN_JRO/1.1 (+'+str(url)+')' + #ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' + self.ckan = RemoteCKAN(url, apikey=Authorization, user_agent=ua) + #self.ckan = RemoteCKAN(url, apikey=Authorization) + self.Authorization = Authorization + if platform.system() == 'Windows': + self.separator = '\\' + else: + self.separator = '/' + + self.chunk_size = 1024 + self.list = [] + self.dict = {} + self.str = '' + self.check = 1 + self.cont = 0 + + def __enter__(self): + return self + + def __exit__(self, *args): + self.ckan.close() + + def action(self, action, **kwargs): + """ + FINALIDAD: + Funcion para llamar a las APIs disponibles + + APIs DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + EJEMPLO: + .action(, param_1 = , ...) + """ + #--------------- CASE: PACKAGE SEARCH ---------------# + if kwargs is not None: + if action == 'package_search': + self.list = ['facet_mincount', 'facet_limit', 'facet_field'] + for facet in self.list: + if facet in kwargs: + kwargs[facet.replace('_', '.')] = kwargs[facet] + kwargs.pop(facet) + #----------------------------------------------------# + try: + return getattr(self.ckan.action, action)(**kwargs) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + + def upload_file(self, dataset_id, file_path, file_date, file_type, **kwargs): + ''' + FINALIDAD: + Funcion para subir un unico archivo al repositorio del ROJ. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .upload_file(dataset_id = , file_date = , file_path = , file_type = , param_1 = , ...) + ''' + self.list = ['package_id', 'upload', 'voc_file_type', 'name'] #file_date + for key1, value1 in kwargs.items(): + if not key1 in self.list: + self.dict[key1] = value1 + + #---------------------------# + if not 'others' in kwargs: + self.dict['others'] = '' + else: + if isinstance(kwargs['others'], list): + self.dict['others'] = json.dumps(kwargs['others']) + #---------------------------# + + if not os.path.isfile(file_path): + return 'File "%s" not exist' % (file_path) + + try: + return getattr(self.ckan.action, 'resource_create')(package_id=dataset_id, file_date=file_date, upload=open(file_path, 'rb'), voc_file_type=file_type, name=file_path.split(self.separator)[-1], **self.dict) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + + def upload_multiple_files(self, dataset_id, path_files, date_files, type_files, **kwargs): + ''' + FINALIDAD: + Funcion para subir multiples archivos al repositorio del ROJ. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .upload_multiple_files(dataset_id = , path_files = or , date_files = or , type_files = or , param_1 = , ...) + ''' + params_dict = {'upload':[], 'name':[]} #ADD FORMAT + #---------------CASO : "path" or "path_list"-----------------# + if type(path_files) is list: + if len(path_files) != 0: + path_files.sort() + for u in path_files: + if os.path.isfile(u): + params_dict['upload'].append(open(u, 'rb')) + params_dict['name'].append(u.split(self.separator)[-1]) + else: + return 'File "%s" not exist' % (u) + else: + return 'ERROR:: "path_list is empty"' + elif type(path_files) is str: + if os.path.isdir(path_files): + path_order = [f for f in os.listdir(path_files) if os.path.isfile(os.path.join(path_files, f))] + path_order.sort() + if path_order: + for name in path_order: + params_dict['upload'].append(open(os.path.join(path_files, name), 'rb')) + params_dict['name'].append(name) + else: + return 'ERROR:: Directory is empty' + else: + return 'ERROR:: Directory "%s" not exist' % (path_files) + else: + return 'ERROR:: "path_files" must be a str or list' + #------------------------------------------------------------# + params_no_dict = {'package_id': dataset_id} + if type(date_files) is list: + params_dict['file_date'] = date_files + else: + params_no_dict['file_date'] = date_files + + if type(type_files) is list: + params_dict['voc_file_type'] = type_files + else: + params_no_dict['voc_file_type'] = type_files + + for key1, value1 in kwargs.items(): + if not key1 in params_dict and not key1 in params_no_dict and key1 != 'others': + if type(value1) is list: + params_dict[key1] = value1 + else: + params_no_dict[key1] = value1 + #------------------------------------------# + if not 'others' in kwargs: + params_no_dict['others'] = '' + else: + if isinstance(kwargs['others'], tuple): + params_dict['others'] = [json.dumps(w) for w in kwargs['others']] + elif isinstance(kwargs['others'], list): + params_no_dict['others'] = json.dumps(kwargs['others']) + elif isinstance(kwargs['others'], str): + params_no_dict['others'] = kwargs['others'] + else: + return 'ERROR:: "others" must be a tuple, list or str' + #------------------------------------------# + len_params_dict = [] + for value2 in params_dict.values(): + len_params_dict.append(len(value2)) + + if len(list(set(len_params_dict))) > 1: + return 'ERROR:: All lists must be the same length: %s' % (len(params_dict['name'])) + #------------------------------------------------------------# + + for v in range(len(params_dict['name'])): + try: + send = {} + for key_dict, value_dict in params_dict.items(): + send[key_dict] = value_dict[v] + for key_no_dict, value_no_dict in params_no_dict.items(): + send[key_no_dict] = value_no_dict + + self.list.append(getattr(self.ckan.action, 'resource_create')(**send)) + print('File "{}" was uploaded successfully'.format(params_dict['name'][v])) + except: + _, exc_value, _ = sys.exc_info() + self.list.append(exc_value) + print('Error uploading "{}" file'.format(params_dict['name'][v])) + return self.list + #------------------------------------------------------------# + + def show(self, type_option, id, **kwargs): + ''' + FINALIDAD: + Funcion personalizada para una busqueda en especifico. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .show(type_option = , id = , param_1 = , ...) + ''' + if type(type_option) is str: + try: + if type_option == 'dataset': + return getattr(self.ckan.action, 'package_show')(id=id, **kwargs) + elif type_option == 'resource': + return getattr(self.ckan.action, 'resource_show')(id=id, **kwargs) + elif type_option == 'project': + return getattr(self.ckan.action, 'organization_show')(id=id, **kwargs) + elif type_option == 'collaborator': + return getattr(self.ckan.action, 'package_collaborator_list_for_user')(id=id, **kwargs) + elif type_option == 'member': + return getattr(self.ckan.action, 'organization_list_for_user')(id=id, **kwargs) + elif type_option == 'vocabulary': + return getattr(self.ckan.action, 'vocabulary_show')(id=id, **kwargs) + elif type_option == 'tag': + if not 'vocabulary_id' in kwargs: + print('Missing "vocabulary_id" value: assume it is a free tag') + return getattr(self.ckan.action, 'tag_show')(id=id, **kwargs) + elif type_option == 'user': + return getattr(self.ckan.action, 'user_show')(id=id, **kwargs) + elif type_option == 'job': + return getattr(self.ckan.action, 'job_show')(id=id, **kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (type_option) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + else: + return 'ERROR:: "type_option" must be a str' + + def search(self, type_option, query=None, **kwargs): + ''' + FINALIDAD: + Funcion personalizada para busquedas que satisfagan algun criterio. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .search(type_option = , query = , param_1 = , ...) + ''' + if type(type_option) is str: + try: + if type_option == 'dataset': + key_replace = ['fq', 'fq_list', 'include_private'] + key_point = ['facet_mincount', 'facet_limit', 'facet_field'] + for key1, value1 in kwargs.items(): + if not key1 in key_replace: + if key1 in key_point: + self.dict[key1.replace('_', '.')] = value1 + else: + self.dict[key1] = value1 + + if query is not None: + if type(query) is dict: + self.dict['fq_list'] = [] + #NUM_RESOURCES_MIN / NUM_RESOURCES_MAX + #----------------------------------------------------# + if 'dataset_start_date' in query: + if type(query['dataset_start_date']) is str: + try: + datetime.strptime(query['dataset_start_date'], '%Y-%m-%d') + if len(query['dataset_start_date']) != 10: + return '"dataset_start_date", must be: ' + self.dict['fq_list'].append('dataset_start_date:"'+query['dataset_start_date']+'"') + self.list.append('dataset_start_date') + except: + return '"dataset_start_date" incorrect: "%s"' % (query['dataset_start_date']) + else: + return '"dataset_start_date" must be ' + #----------------------------------------------------# + if 'dataset_end_date' in query: + if type(query['dataset_end_date']) is str: + try: + datetime.strptime(query['dataset_end_date'], '%Y-%m-%d') + if len(query['dataset_end_date']) != 10: + return '"dataset_end_date", must be: ' + + if 'dataset_start_date' in query: + if query['dataset_start_date'] > query['dataset_end_date']: + return '"dataset_end_date" must be greater than "dataset_start_date"' + + self.dict['fq_list'].append('dataset_end_date:"'+query['dataset_end_date']+'"') + self.list.append('dataset_end_date') + except: + return '"dataset_end_date" incorrect: "%s"' % (query['dataset_end_date']) + else: + return '"dataset_end_date" must be ' + #----------------------------------------------------# + for key, value in query.items(): + if value is not None and not key in self.list: + self.dict['fq_list'].append(str(key)+':"'+str(value)+'"') + else: + return '"query" must be ' + + return getattr(self.ckan.action, 'package_search')(include_private=True, **self.dict) + + elif type_option == 'resource': + for key1, value1 in kwargs.items(): + if key1 != 'fields': + self.dict[key1] = value1 + + if query is not None: + if type(query) is dict: + #----------------------------------------------------# + if 'file_date_min' in query: + if type(query['file_date_min']) is str: + try: + datetime.strptime(query['file_date_min'], '%Y-%m-%d') + if len(query['file_date_min']) != 10: + return '"file_date_min", must be: ' + except: + return '"file_date_min" incorrect: "%s"' % (query['file_date_min']) + else: + return '"file_date_min" must be ' + #----------------------------------------------------# + if 'file_date_max' in query: + if type(query['file_date_max']) is str: + try: + datetime.strptime(query['file_date_max'], '%Y-%m-%d') + if len(query['file_date_max']) != 10: + return '"file_date_max", must be: ' + + if 'file_date_min' in query: + if query['file_date_min'] > query['file_date_max']: + return '"file_date_max" must be greater than "file_date_min"' + except: + return '"file_date_max" incorrect: "%s"' % (query['file_date_max']) + else: + return '"file_date_max" must be ' + #----------------------------------------------------# + self.dict['query'] = query + else: + return '"query" must be ' + return getattr(self.ckan.action, 'resources_search')(**self.dict) + + elif type_option == 'tag': + for key1, value1 in kwargs.items(): + if key1 != 'fields': + self.dict[key1] = value1 + + if not 'vocabulary_id' in kwargs: + print('Missing "vocabulary_id" value: tags that don’t belong to any vocabulary') + else: + print('Only tags that belong to "{}" vocabulary'.format(kwargs['vocabulary_id'])) + + if query is not None: + if type(query) is dict: + if 'search' in query: + if type(query['search']) is list or type(query['search']) is str: + self.dict['query'] = query['search'] + else: + return '"search" must be or ' + else: + return '"query" must be ' + return getattr(self.ckan.action, 'tag_search')(**self.dict) + + else: + return 'ERROR:: "%s" is not accepted' % (type_option) + + except: + _, exc_value, _ = sys.exc_info() + return exc_value + else: + return 'ERROR:: "type_option" must be ' + + def create(self, type_option, **kwargs): + ''' + FINALIDAD: + Funcion personalizada para crear. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .create(type_option = , param_1 = , ...) + ''' + if type(type_option) is str: + try: + if type_option == 'dataset': + return getattr(self.ckan.action, 'package_create')(**kwargs) + elif type_option == 'project': + return getattr(self.ckan.action, 'organization_create')(**kwargs) + elif type_option == 'member': + return getattr(self.ckan.action, 'organization_member_create')(**kwargs) + elif type_option == 'collaborator': + return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) + elif type_option == 'vocabulary': + return getattr(self.ckan.action, 'vocabulary_create')(**kwargs) + elif type_option == 'tag': + return getattr(self.ckan.action, 'tag_create')(**kwargs) + elif type_option == 'user': + return getattr(self.ckan.action, 'user_create')(**kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (type_option) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + else: + return 'ERROR:: "type_option" must be ' + + def patch(self, type_option, **kwargs): + ''' + FINALIDAD: + Funciones personalizadas para actualizar + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .patch(type_option = , param_1 = , ...) + ''' + if type(type_option) is str: + try: + if type_option == 'dataset': + return getattr(self.ckan.action, 'package_patch')(**kwargs) + elif type_option == 'project': + return getattr(self.ckan.action, 'organization_patch')(**kwargs) + elif type_option == 'resource': + return getattr(self.ckan.action, 'resource_patch')(**kwargs) + elif type_option == 'member': + return getattr(self.ckan.action, 'organization_member_create')(**kwargs) + elif type_option == 'collaborator': + return getattr(self.ckan.action, 'package_collaborator_create')(**kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (type_option) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + else: + return 'ERROR:: "type_option" must be ' + + def delete(self, type_option, select=None, **kwargs): + ''' + FINALIDAD: + Función personalizada para eliminar y/o purgar. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .delete(type_option = , param_1 = , ...) + ''' + if type(type_option) is str: + try: + if type_option == 'dataset': + if select is None: + return 'ERROR:: "select" must not be "None"' + else: + if 'delete' == select: + return getattr(self.ckan.action, 'package_delete')(**kwargs) + elif 'purge' == select: + return getattr(self.ckan.action, 'dataset_purge')(**kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (select) + elif type_option == 'project': + if select is None: + return 'ERROR:: "select" must not be "None"' + else: + if 'delete' == select: + return getattr(self.ckan.action, 'organization_delete')(**kwargs) + elif 'purge' == select: + return getattr(self.ckan.action, 'organization_purge')(**kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (select) + elif type_option == 'resource': + return getattr(self.ckan.action, 'resource_delete')(**kwargs) + elif type_option == 'vocabulary': + return getattr(self.ckan.action, 'vocabulary_delete')(**kwargs) + elif type_option == 'tag': + return getattr(self.ckan.action, 'tag_delete')(**kwargs) + elif type_option == 'user': + return getattr(self.ckan.action, 'user_delete')(**kwargs) + else: + return 'ERROR:: "%s" is not accepted' % (type_option) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + else: + return 'ERROR:: "type_option" must be ' + + def f_status_note(self, total, result, path): + file_txt = open(path+'status_note.txt', 'w') + file_txt = open(path+'status_note.txt', 'a') + + file_txt.write('DOWNLOADED FILE(S): "%s"' % (len(result['name']))) + file_txt.write(''+ os.linesep) + for u in result['name']: + file_txt.write(' - '+ u + os.linesep) + file_txt.write(''+ os.linesep) + + file_txt.write('FAILED FILE(S): "%s"' % (len(total['name'])-len(result['name']))) + file_txt.write(''+ os.linesep) + if len(total['name'])-len(result['name']) != 0: + for u in total['name']: + if not u in result['name']: + file_txt.write(' - '+ u + os.linesep) + else: + file_txt.write(' "None"'+ os.linesep) + + def f_name(self, name_dataset, ext, tempdir): + while self.check: + self.str = '' + if self.cont == 0: + if os.path.exists(tempdir + name_dataset + ext): + self.str = name_dataset+'('+str(self.cont+1)+')'+ext + else: + self.check = self.check * 0 + self.str = name_dataset + ext + else: + if not os.path.exists(tempdir + name_dataset+'('+str(self.cont)+')'+ext): + self.check = self.check * 0 + self.str = name_dataset+'('+str(self.cont)+')'+ ext + self.cont = self.cont+1 + return self.str + + def f_zipdir(self, path, ziph, zip_name): + for root, _, files in os.walk(path): + print('.....') + print('Creating: "{}" >>'.format(zip_name)) + for __file in tqdm(iterable=files, total=len(files)): + new_dir = os.path.relpath(os.path.join(root, __file), os.path.join(path, '..')) + ziph.write(os.path.join(root, __file), new_dir) + print('Created >>') + + def download_by_step(self, response, tempdir_name): + try: + with requests.get(response['url'], stream=True, headers={'Authorization': self.Authorization}) as resp: + if resp.status_code == 200: + with open(tempdir_name+response['name'], 'wb') as file: + for chunk in resp.iter_content(chunk_size = self.chunk_size): + if chunk: + file.write(chunk) + except requests.exceptions.RequestException: + pass + + def download_files(self, **kwargs): + ''' + FINALIDAD: + Funcion personalizada para la descarga de archivos existentes de un dataset. + + PARAMETROS DISPONIBLES: + CONSULTAR: "GUIA DE SCRIPT.pdf" + + ESTRUCTURA: + .download_files(id = , param_1 = , ...) + ''' + dict_local = {} + #----------------------------------------------# + if 'zip' in kwargs: + if type(kwargs['zip']) is not bool: + return 'ERROR:: "zip" must be: ' + else: + dict_local['zip'] = kwargs['zip'] + else: + dict_local['zip'] = False + #----------------------------------------------# + if 'status_note' in kwargs: + if type(kwargs['status_note']) is not bool: + return 'ERROR:: "status_note" must be: ' + else: + dict_local['status_note'] = kwargs['status_note'] + else: + dict_local['status_note'] = False + #----------------------------------------------# + if 'path' in kwargs: + if type(kwargs['path']) is str: + if os.path.isdir(kwargs['path']) == False: + return 'ERROR:: "path" does not exist' + else: + if kwargs['path'][-1:] != self.separator: + dict_local['path'] = kwargs['path']+self.separator + else: + dict_local['path'] = kwargs['path'] + + txt = dict_local['path']+datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")+'.txt' + if int(platform.python_version()[0]) == 3: + try: + file_txt = open(txt, 'w') + file_txt.close() + os.remove(txt) + except PermissionError: + return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) + else: + try: + file_txt = open(txt, 'w') + file_txt.close() + os.remove(txt) + except: + return 'ERROR:: Access denied, you are not authorized to write files: "%s"' % (dict_local['path']) + else: + return 'ERROR:: "path" must be: ' + else: + dict_local['path'] = '' + #----------------------------------------------# + for key, value in kwargs.items(): + if not key in dict_local: + self.dict[key] = value + try: + response = getattr(self.ckan.action, 'url_resources')(**self.dict) + except: + _, exc_value, _ = sys.exc_info() + return exc_value + + if len(response) != 0: + #--------------TEMP PATH---------------# + if dict_local['zip']: + tempdir = tempfile.mkdtemp(prefix=kwargs['id']+'-')+self.separator + os.mkdir(tempdir+kwargs['id']) + dir_name = tempdir + kwargs['id'] + self.separator + else: + dir = self.f_name(kwargs['id'], '', dict_local['path']) + os.mkdir(dict_local['path'] + dir) + dir_name = dict_local['path'] + dir + self.separator + #-----------DOWNLOAD FILES-------------# + print('.....') + print('Downloading "{}" file(s) >>'.format(len(response))) + name_total = {'name': []} + with concurrent.futures.ThreadPoolExecutor() as executor: + for u in tqdm(iterable=response, total=len(response)): + name_total['name'].append(u['name']) + executor.submit(self.download_by_step, u, dir_name) + name_check = {} + name_check['name'] = [f for f in os.listdir(dir_name) if os.path.isfile(os.path.join(dir_name, f))] + print('"{}" downloaded file(s) successfully >>'.format(len(name_check['name']))) + #--------------------------------------# + if len(name_check['name']) != 0: + #----------Status Note---------# + if dict_local['status_note']: + print('.....') + print('Creating: "status_note.txt" >>') + self.f_status_note(name_total, name_check, dir_name) + print('Created>>') + #----------ZIP CREATE----------# + if dict_local['zip']: + zip_name = self.f_name(kwargs['id'], '.zip', dict_local['path']) + ziph = zipfile.ZipFile(dict_local['path'] + zip_name, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) + self.f_zipdir(dir_name, ziph, zip_name) + ziph.close() + #Delete Temporal Path + if os.path.exists(tempdir[:-1]): + shutil.rmtree(tempdir[:-1]) + #------------------------------# + print('.....') + return 'DOWNLOAD FINISHED' + else: + #Delete Temporal Path + if dict_local['zip']: + if os.path.exists(tempdir[:-1]): + shutil.rmtree(tempdir[:-1]) + else: + if os.path.exists(dir_name[:-1]): + shutil.rmtree(dir_name[:-1]) + return 'NO FILES WERE DOWNLOADED' + else: + return 'FILES NOT FOUND' \ No newline at end of file diff --git a/script/setup.py b/script/setup.py new file mode 100755 index 0000000..26bfc2c --- /dev/null +++ b/script/setup.py @@ -0,0 +1,12 @@ +# encoding: utf-8 +from setuptools import setup + +setup( + name = "CKAN_JRO", + version = "master", + description = "Data Repository - JRO", + author = "Edson Ynilupu Mattos", + author_email = "eynilupu@igp.gob.pe", + url = "", + packages = ["CKAN_JRO"] +) \ No newline at end of file