Skip to content

Commit c00949d

Browse files
tacxoutacxou
committed
chore: Update .gitignore and requirements.txt, and configure VSCode settings
1 parent 7e791b7 commit c00949d

File tree

6 files changed

+173
-86
lines changed

6 files changed

+173
-86
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
cache/
2+
cache_old/
23
logs/
34
.env
45
src/__pycache__/*
56
.idea/
7+
lib/
8+
bin/
9+
pyvenv.cfg

‎.vscode/settings.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
{
22
"nuxt.isNuxtApp": false,
3-
"python.analysis.extraPaths": ["./src"]
3+
"python.analysis.extraPaths": ["./src"],
4+
"python.analysis.autoImportCompletions": true,
5+
"python.analysis.typeCheckingMode": "basic"
46
}

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ build: ## Construit l'image docker
1818
@docker build -t sesame-taiga_crawler .
1919
@printf "\033[33mDOCKER:\033[0m SUCCESSFUL !!!\n"
2020

21+
pull-crawler-docker: ## Pull l'image docker
22+
@docker pull $(IMG_NAME)
23+
2124
run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
2225
@docker run --rm -it \
2326
--add-host host.docker.internal:host-gateway \

‎config.yml‎

Lines changed: 157 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,157 @@
1-
{
2-
"taiga_etd.json": {
3-
"mapping": {
4-
"inetOrgPerson.cn": "nom",
5-
"inetOrgPerson.sn": "prenom",
6-
"inetOrgPerson.givenName": "prenom",
7-
"additionalFields.attributes.supannPerson.supannPrenomsEtatCivil": "prenom",
8-
"$setOnInsert.inetOrgPerson.uid": "email1",
9-
"inetOrgPerson.employeeNumber": "id_coord",
10-
"additionalFields.attributes.supannPerson.supannRefId": "id_coord",
11-
"inetOrgPerson.mail": "email1",
12-
"inetOrgPerson.mobile": "tel_mob",
13-
"inetOrgPerson.postalAddress": "adresse",
14-
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
15-
"additionalFields.attributes.supannPerson.supanncivilite": "civilite",
16-
"additionalFields.attributes.supannPerson.supannOIDCGenre": "civilite",
17-
"additionalFields.attributes.supannPerson.supannNomdeNaissance": "nom_marital",
18-
"additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance": "nss_date",
19-
"additionalFields.attributes.supannPerson.supannCodeINSEEPaysDeNaissance": "nns_pays",
20-
"additionalFields.attributes.supannPerson.supannCodeINSEEVilleDeNaissance": "nss_ville",
21-
"additionalFields.attributes.supannPerson.supannAutreMail": "email2"
22-
},
23-
"additionalFields": {
24-
"additionalFields.objectClasses": ["supannPerson"],
25-
"additionalFields.attributes.supannPerson.supannTypeEntiteAffectation": "etd",
26-
"state": -1
27-
}
28-
},
29-
"taiga_adm.json": {
30-
"mapping": {
31-
"inetOrgPerson.cn": "nom",
32-
"inetOrgPerson.sn": "prenom",
33-
"inetOrgPerson.givenName": "prenom",
34-
"additionalFields.attributes.supannPerson.supannPrenomsEtatCivil": "prenom",
35-
"$setOnInsert.inetOrgPerson.uid": "email1",
36-
"inetOrgPerson.employeeNumber": "id_coord",
37-
"additionalFields.attributes.supannPerson.supannEmpId": "id_coord",
38-
"inetOrgPerson.mail": "email1",
39-
"inetOrgPerson.mobile": "tel_mob",
40-
"inetOrgPerson.postalAddress": "adresse",
41-
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
42-
"additionalFields.attributes.supannPerson.supanncivilite": "civilite",
43-
"additionalFields.attributes.supannPerson.supannOIDCGenre": "civilite",
44-
"additionalFields.attributes.supannPerson.supannNomdeNaissance": "nom_marital",
45-
"additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance": "nss_date",
46-
"additionalFields.attributes.supannPerson.supannCodeINSEEPaysDeNaissance": "nns_pays",
47-
"additionalFields.attributes.supannPerson.supannCodeINSEEVilleDeNaissance": "nss_ville",
48-
"additionalFields.attributes.supannPerson.supannAutreMail": "email2"
49-
},
50-
"additionalFields": {
51-
"additionalFields.objectClasses": ["supannPerson"],
52-
"additionalFields.attributes.supannPerson.supannTypeEntiteAffectation": "adm",
53-
"state": -1
54-
}
55-
},
56-
"taiga_esn.json": {
57-
"mapping": {
58-
"inetOrgPerson.cn": "nom",
59-
"inetOrgPerson.sn": "prenom",
60-
"inetOrgPerson.givenName": "prenom",
61-
"additionalFields.attributes.supannPerson.supannPrenomsEtatCivil": "prenom",
62-
"$setOnInsert.inetOrgPerson.uid": "email1",
63-
"inetOrgPerson.employeeNumber": "id_coord",
64-
"additionalFields.attributes.supannPerson.supannEmpId": "id_coord",
65-
"inetOrgPerson.mail": "email1",
66-
"inetOrgPerson.mobile": "tel_mob",
67-
"inetOrgPerson.postalAddress": "adresse",
68-
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
69-
"additionalFields.attributes.supannPerson.supanncivilite": "civilite",
70-
"additionalFields.attributes.supannPerson.supannOIDCGenre": "civilite",
71-
"additionalFields.attributes.supannPerson.supannNomdeNaissance": "nom_marital",
72-
"additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance": "nss_date",
73-
"additionalFields.attributes.supannPerson.supannCodeINSEEPaysDeNaissance": "nns_pays",
74-
"additionalFields.attributes.supannPerson.supannCodeINSEEVilleDeNaissance": "nss_ville",
75-
"additionalFields.attributes.supannPerson.supannAutreMail": "email2"
76-
},
77-
"additionalFields": {
78-
"additionalFields.objectClasses": ["supannPerson"],
79-
"additionalFields.attributes.supannPerson.supannTypeEntiteAffectation": "esn",
80-
"state": -1
81-
}
82-
}
83-
}
1+
---
2+
taiga_etd.json:
3+
mapping:
4+
inetOrgPerson.cn:
5+
- "nom"
6+
- "prenom"
7+
inetOrgPerson.displayName:
8+
- "prenom"
9+
- "nom"
10+
inetOrgPerson.sn: "nom"
11+
inetOrgPerson.givenName: "prenom"
12+
additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
13+
inetOrgPerson.employeeNumber: "id_coord"
14+
"$setOnInsert.inetOrgPerson.uid":
15+
- "prenom"
16+
- "nom"
17+
additionalFields.attributes.supannPerson.supannRefId: "id_coord"
18+
"$setOnInsert.inetOrgPerson.mail":
19+
- "prenom"
20+
- "nom"
21+
inetOrgPerson.mobile: "tel_mob"
22+
inetOrgPerson.postalAddress:
23+
- "adresse"
24+
- "CP"
25+
- "Ville"
26+
additionalFields.attributes.supannPerson.supanncivilite: "civilite"
27+
additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
28+
additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
29+
additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
30+
additionalFields.attributes.supannPerson.supannAutreMail: "email2"
31+
additionalFields:
32+
additionalFields.objectClasses:
33+
- "supannPerson"
34+
additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "etd"
35+
state: -1
36+
inetOrgPerson.employeeType: "TAIGA"
37+
inetOrgPerson.departmentNumber: "etd"
38+
additionalFields.attributes.supannPerson.supannEtablissement: "{UAI}0690184A"
39+
transforms:
40+
inetOrgPerson.cn: "join(delimiter=' ')"
41+
inetOrgPerson.displayName: "join(delimiter=' ')"
42+
inetOrgPerson.postalAddress: "join(delimiter=',')"
43+
"$setOnInsert.inetOrgPerson.uid":
44+
- "join(delimiter='.')"
45+
- "remove_accents"
46+
- "lower"
47+
- "replace(old=' ',new='-')"
48+
# - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
49+
"$setOnInsert.inetOrgPerson.mail":
50+
- "join(delimiter='.')"
51+
- "lower"
52+
- "remove_accents"
53+
- "replace(old=' ',new='-')"
54+
- "suffix(string='@sesame.local')"
55+
# taiga_adm.json:
56+
# mapping:
57+
# inetOrgPerson.cn:
58+
# - "nom"
59+
# - "prenom"
60+
# inetOrgPerson.displayName:
61+
# - "prenom"
62+
# - "nom"
63+
# inetOrgPerson.sn: "nom"
64+
# inetOrgPerson.givenName: "prenom"
65+
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
66+
# inetOrgPerson.employeeNumber: "id_coord"
67+
# $setOnInsert.inetOrgPerson.uid:
68+
# - "prenom"
69+
# - "nom"
70+
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
71+
# $setOnInsert.inetOrgPerson.mail:
72+
# - "prenom"
73+
# - "nom"
74+
# inetOrgPerson.mobile: "tel_mob"
75+
# inetOrgPerson.postalAddress:
76+
# - "adresse"
77+
# - "CP"
78+
# - "Ville"
79+
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
80+
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
81+
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
82+
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
83+
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
84+
# additionalFields:
85+
# additionalFields.objectClasses:
86+
# - "supannPerson"
87+
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "adm"
88+
# inetOrgPerson.employeeType: "TAIGA"
89+
# inetOrgPerson.departmentNumber: "adm"
90+
# state: -1
91+
# transforms:
92+
# inetOrgPerson.cn: "join(delimiter=' ')"
93+
# inetOrgPerson.displayName: "join(delimiter=' ')"
94+
# inetOrgPerson.postalAddress: "join(delimiter=',')"
95+
# $setOnInsert.inetOrgPerson.uid:
96+
# - "join(delimiter='.')"
97+
# - "remove_accents"
98+
# - "lower"
99+
# - "replace(old=' ',new='-')"
100+
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
101+
# $setOnInsert.inetOrgPerson.mail:
102+
# - "join(delimiter='.')"
103+
# - "remove_accents"
104+
# - "lower"
105+
# - "replace(old=' ',new='-')"
106+
# - "suffix(string='@sesame.local')"
107+
# taiga_esn.json:
108+
# mapping:
109+
# inetOrgPerson.cn:
110+
# - "nom"
111+
# - "prenom"
112+
# inetOrgPerson.displayName:
113+
# - "prenom"
114+
# - "nom"
115+
# inetOrgPerson.sn: "nom"
116+
# inetOrgPerson.givenName: "prenom"
117+
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
118+
# inetOrgPerson.employeeNumber: "id_coord"
119+
# $setOnInsert.inetOrgPerson.uid:
120+
# - "prenom"
121+
# - "nom"
122+
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
123+
# $setOnInsert.inetOrgPerson.mail:
124+
# - "prenom"
125+
# - "nom"
126+
# inetOrgPerson.mobile: "tel_mob"
127+
# inetOrgPerson.postalAddress:
128+
# - "adresse"
129+
# - "CP"
130+
# - "Ville"
131+
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
132+
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
133+
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
134+
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
135+
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
136+
# additionalFields:
137+
# additionalFields.objectClasses:
138+
# - "supannPerson"
139+
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "esn"
140+
# inetOrgPerson.departmentNumber: "esn"
141+
# inetOrgPerson.employeeType: "TAIGA"
142+
# state: -1
143+
# transforms:
144+
# inetOrgPerson.cn: "join(delimiter=' ')"
145+
# inetOrgPerson.displayName: "join(delimiter=' ')"
146+
# inetOrgPerson.postalAddress: "join(delimiter=',')"
147+
# $setOnInsert.inetOrgPerson.uid:
148+
# - "join(delimiter='.')"
149+
# - "remove_accents"
150+
# - "lower"
151+
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
152+
# $setOnInsert.inetOrgPerson.mail:
153+
# - "join(delimiter='.')"
154+
# - "remove_accents"
155+
# - "lower"
156+
# - "replace(old=' ',new='-')"
157+
# - "suffix(string='@sesame.local')"

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
aiohttp==3.9.3
2-
DataWeaver==1.0.6
2+
DataWeaver==1.0.7
33
python-dotenv==1.0.1
44
Requests==2.31.0
55
urllib3==1.26.18

‎src/import_ind.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ async def send_request(session, url, json):
4141

4242
async with session.post(url, json=json, headers=headers, params=params) as response:
4343
print(f"Request to {url} successful: {response.status}")
44-
await read_response(response)
44+
if response.status == 304:
45+
print(f"Cached entry {json.get('inetOrgPerson', {}).get('employeeNumber')}")
46+
else:
47+
print(f"Response to {json.get('inetOrgPerson', {}).get('employeeNumber')}:")
48+
await read_response(response)
4549
response.raise_for_status() # Raises error for 4xx/5xx responses
4650
except aiohttp.ClientResponseError as e:
4751
# This catches responses like 400, 404, 500 etc.

0 commit comments

Comments
 (0)