Commit 7969e9ba authored by Skia's avatar Skia

Update SAS migration script

parent 47cb5e60
...@@ -1029,8 +1029,9 @@ def migrate_sas(): ...@@ -1029,8 +1029,9 @@ def migrate_sas():
album_link = {} album_link = {}
picture_link = {} picture_link = {}
FILE_ROOT = "/data/sas/" FILE_ROOT = "/data/sas/"
SithFile.objects.filter(id__gt=18886).delete() SithFile.objects.filter(id__gte=18892).delete()
print("Album/Pictures deleted") print("Album/Pictures deleted")
reset_index('core', 'sas')
cur = db.cursor(MySQLdb.cursors.SSDictCursor) cur = db.cursor(MySQLdb.cursors.SSDictCursor)
cur.execute(""" cur.execute("""
SELECT * SELECT *
...@@ -1058,61 +1059,75 @@ def migrate_sas(): ...@@ -1058,61 +1059,75 @@ def migrate_sas():
except: pass except: pass
print("Album migrated at %s" % datetime.datetime.now()) print("Album migrated at %s" % datetime.datetime.now())
print("Running time: %s" % (datetime.datetime.now()-start)) print("Running time: %s" % (datetime.datetime.now()-start))
cur.execute(""" with open("albums.link", "w") as f:
SELECT * f.write(str(album_link))
FROM sas_photos cur.close()
""") finished = False
for r in cur: chunk = 0
try: while not finished:
user = User.objects.filter(id=r['id_utilisateur']).first() or root cur = db.cursor(MySQLdb.cursors.SSDictCursor)
parent = Album.objects.filter(id=album_link[str(r['id_catph'])]).first() cur.execute("""
SELECT *
FROM sas_photos
ORDER BY 'id_photo'
LIMIT %s, 1000
""", (chunk*1000, ))
has_result = False
for r in cur:
try:
user = User.objects.filter(id=r['id_utilisateur']).first() or root
parent = Album.objects.filter(id=album_link[str(r['id_catph'])]).first()
file_name = FILE_ROOT file_name = FILE_ROOT
if r['date_prise_vue']: if r['date_prise_vue']:
file_name += r['date_prise_vue'].strftime("%Y/%m/%d") file_name += r['date_prise_vue'].strftime("%Y/%m/%d")
else: else:
file_name += '/'.join(["1970", "01", "01"]) file_name += '/'.join(["1970", "01", "01"])
file_name += "/" + str(r['id_photo']) + ".jpg" file_name += "/" + str(r['id_photo']) + ".jpg"
file = File(open(file_name, "rb")) file = File(open(file_name, "rb"))
file.name = str(r['id_photo']) + ".jpg" file.name = (str(r['id_photo']) + ".jpg").encode('utf-8')
p = Picture( p = Picture(
name=to_unicode(str(r['id_photo'])) + ".jpg", name=(to_unicode(str(r['id_photo'])) + ".jpg").encode('utf-8'),
owner=user, owner=user,
is_moderated=True, is_moderated=True,
is_folder=False, is_folder=False,
mime_type="image/jpeg", mime_type="image/jpeg",
parent=parent, parent=parent,
file=file, file=file,
) )
if r['date_prise_vue']: if r['date_prise_vue']:
p.date = r['date_prise_vue'].replace(tzinfo=timezone('Europe/Paris')) p.date = r['date_prise_vue'].replace(tzinfo=timezone('Europe/Paris'))
else: else:
p.date = r['date_ajout_ph'].replace(tzinfo=timezone('Europe/Paris')) p.date = r['date_ajout_ph'].replace(tzinfo=timezone('Europe/Paris'))
for f in p._meta.local_fields: for f in p._meta.local_fields:
if f.name == "date": if f.name == "date":
f.auto_now = False f.auto_now = False
p.generate_thumbnails() p.generate_thumbnails()
db2 = MySQLdb.connect(**settings.OLD_MYSQL_INFOS) p.save()
cur2 = db2.cursor(MySQLdb.cursors.SSDictCursor) db2 = MySQLdb.connect(**settings.OLD_MYSQL_INFOS)
cur2.execute(""" cur2 = db2.cursor(MySQLdb.cursors.SSDictCursor)
SELECT * cur2.execute("""
FROM sas_personnes_photos SELECT *
WHERE id_photo = %s FROM sas_personnes_photos
""", (r['id_photo'], )) WHERE id_photo = %s
for r2 in cur2: """, (r['id_photo'], ))
try: for r2 in cur2:
u = User.objects.filter(id=r2['id_utilisateur']).first() try:
if u: u = User.objects.filter(id=r2['id_utilisateur']).first()
PeoplePictureRelation(user=u, picture=p).save() if u:
except: PeoplePictureRelation(user=u, picture=p).save()
print("Fail to associate user %d to picture %d" % (r2['id_utilisateur'], p.id)) except:
picture_link[str(r['id_photo'])] = p.id print("Fail to associate user %d to picture %d" % (r2['id_utilisateur'], p.id))
except Exception as e: has_result = True
pass except Exception as e:
print("FAIL to migrate Picture: %s" % (repr(e))) print("FAIL to migrate Picture: %s" % (repr(e)))
cur.close() cur.close()
print("Chunk %d migrated at %s" % (chunk, str(datetime.datetime.now())))
print("Running time: %s" % (datetime.datetime.now()-start))
chunk += 1
finished = not has_result
print("SAS migrated at %s" % datetime.datetime.now()) print("SAS migrated at %s" % datetime.datetime.now())
print("Running time: %s" % (datetime.datetime.now()-start)) print("Running time: %s" % (datetime.datetime.now()-start))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment