How to import data of Google+ to Joplin ?

99 x served & 54 x viewed

Install JOPLIN : https://joplin.cozic.net ,  and start REST API.

Step 1 : Download all with https://takeout.google.com

Step 2 : Uncompress and put all on same folder.

Step 3 : Put this script in folder.

Step 4 : Edit the script and put your token

The script :

#
# Version 1 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#


from os import listdir
from pathlib import Path
import glob
import csv
import locale
import os
import time
from datetime import datetime
import json
import requests

nb_metadata = 0
nb_metadata_import = 0
def month_string_to_number(string):
    m = {
        'janv.': 1,
        'feb.': 2,
        'févr.': 2,
        'mar.': 3,
        'mars': 3,
        'apr.':4,
        'avr.':4,
         'may.':5,
         'mai':5,
         'juin':6,
         'juil.':7,
         'aug.':8,
         'août':8,
         'sept.':9,
         'oct.':10,
         'nov.':11,
         'déc.':12
        }
    s = string.strip()[:5].lower()

    try:
        out = m[s]
        return out
    except:
        raise ValueError('Not a month')

locale.setlocale(locale.LC_TIME, 'fr_FR.UTF-8')
#today = datetime.date.today()
#print(today.strftime('The date :%d %b. %Y à %H:%M:%S UTC'))
from time import strftime,localtime
print(localtime())
print(strftime("%H:%M:%S, %d %b. %Y",localtime()))
date = datetime.strptime('2017-05-04',"%Y-%m-%d")

#Token
ip = "127.0.0.1"
port = "41184"
token = "Put your token here"

nb_import = 0;
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}

url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
url_folders = (
    "http://"+ip+":"+port+"/folders?"
    "token="+token
)
url_tags = (
    "http://"+ip+":"+port+"/tags?"
    "token="+token
)
url_ressources = (
    "http://"+ip+":"+port+"/ressources?"
    "token="+token
)

#Init
GooglePlus_UID = "12345678901234567801234567890123"
UID = {}

payload = {
    "id":GooglePlus_UID,
    "title":"GooglePlus Import"
}

try:
    resp = requests.post(url_folders, data=json.dumps(payload, separators=(',',':')), headers=headers)
    resp.raise_for_status()
    resp_dict = resp.json()
    print(resp_dict)
    print("My ID")
    print(resp_dict['id'])
    GooglePlus_UID_real = resp_dict['id']
    save = str(resp_dict['id'])
    UID[GooglePlus_UID]= save
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

for csvfilename in glob.iglob('Takeout*/**/*.metadata.csv', recursive=True):
  nb_metadata += 1
  print(nb_metadata," ",csvfilename)
  #print("Picture:"+os.path.basename(csvfilename))
  mybasename = os.path.basename(csvfilename)
  mylist = mybasename.split(".")
  myfilename = mylist[0] + "." + mylist[1]
  filename = os.path.dirname(csvfilename)+"/"+myfilename
  my_file = Path(filename)
  with open(csvfilename) as csvfile:
    reader = csv.DictReader(csvfile)
    for row in reader:
        if (len(row['description']) > 0):
            print(row['title'], row['description'], row['creation_time.formatted'], row['geo_data.latitude'], row['geo_data.longitude'])
            #date = datetime.strptime(row['creation_time.formatted'], "%d %b %Y à %H:%M:%S %Z").timetuple()
            #print(date)
            mylist2 = row['creation_time.formatted'].split(" ");
            mylist3 = mylist2[4].split(":");
            date = date.replace(hour=int(mylist3[0]), year=int(mylist2[2]), month=month_string_to_number(mylist2[1]), day=int(mylist2[0]))
            timestamp = time.mktime(date.timetuple())*1000
            print(timestamp)
            nb_metadata_import += 1
            mybody = row['description']
            if (len(row['geo_data.latitude']) > 2):
              payload_note = {
                "parent_id":GooglePlus_UID_real,
                "title":row['creation_time.formatted'],
                "source":myfilename,
                "source_url":row['url'],
                "order":nb_metadata_import,
                "body":mybody
                }
              payload_note_put = {
                "latitude":float(row['geo_data.latitude']),
                "longitude":float(row['geo_data.longitude']),
                "source":myfilename,
                "source_url":row['url'],
                "order":nb_metadata_import,
                "user_created_time":timestamp,
                "user_updated_time":timestamp,
                "author":"Google+"
                }
            else:
               payload_note = {
                "parent_id":GooglePlus_UID_real,
                "title":row['creation_time.formatted'],
                "source":myfilename,
                "source_url":row['url'],
                "order":nb_metadata_import,
                "user_created_time":timestamp,
                "user_updated_time":timestamp,
                "author":"Google+",
                "body":mybody
                }
               payload_note_put = {
                "source":myfilename,
                "order":nb_metadata_import,
                "source_url":row['url'],
                "user_created_time":timestamp,
                "user_updated_time":timestamp,
                "author":"Google+"
                }

            try:
                resp = requests.post(url_notes, json=payload_note)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                myuid= resp_dict['id']
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

            url_notes_put = (
    "http://"+ip+":"+port+"/notes/"+myuid+"?"
    "token="+token
)

            try:
                resp = requests.put(url_notes_put, json=payload_note_put)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)
            
            if my_file.is_file():
               cmd = "curl -F 'data=@"+filename+"' -F 'props={\"title\":\""+myfilename+"\"}' http://"+ip+":"+port+"/resources?token="+token
               print("Command"+cmd)
               resp = os.popen(cmd).read()
               try:
                  respj = json.loads(resp)
                  print(respj['id'])
                  myuid_picture= respj['id']
               except:
                  print('bad json: ', resp)

               mybody = row['description'] + "\n  ![" + myfilename + "](:/" + myuid_picture + ")   \n";

               payload_note_put = {
                "body":mybody
                }

               try:
                  resp = requests.put(url_notes_put, json=payload_note_put)
                  resp.raise_for_status()
                  resp_dict = resp.json()
                  print(resp_dict)
               except requests.exceptions.HTTPError as e:
                  print("Bad HTTP status code:", e)
               except requests.exceptions.RequestException as e:
                  print("Network error:", e)

print(nb_metadata)
print(nb_metadata_import)

Diaro App : DiaroBackup.xml : How to parse in python ? (Draft n°4)

71 x served & 19 x viewed

(See the finale release : http://www.cyber-neurones.org/2019/02/diaro-app-pixel-crater-ltd-diarobackup-xml-how-to-migrate-data-to-joplin/ )

Je pensais avoir trouvé mon bug … je pensais que c’était l’espace avant la variable qui faisait que les valeurs n’était pas prise en compte.

Avec : requests.post(url_folders, json=payload) on a un . c.a.d. un 20

Avec : requests.post(url_folders, data=json.dumps(payload, separators=(‘,’,’:’)), headers=headers)

Sachant que : headers = {‘Content-type’: ‘application/json’, ‘Accept’: ‘text/plain’}

Je n’ai plus d’espace mais les valeurs numériques ne sont pas prise en compte …

Mon dernier code de migration : V4 :

#
# Version 4 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

import xml.etree.ElementTree as etree
from time import gmtime, strftime
import time
import json
import requests
import os

strftime("%Y-%m-%d %H:%M:%S", gmtime())
start = time.time()

#Token
ip = "127.0.0.1"
port = "41184"
token = "ABCD123ABCD123ABCD123ABCD123ABCD123"
nb_import = 0;
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}

url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
url_folders = (
    "http://"+ip+":"+port+"/folders?"
    "token="+token
)
url_tags = (
    "http://"+ip+":"+port+"/tags?"
    "token="+token
)
url_ressources = (
    "http://"+ip+":"+port+"/ressources?"
    "token="+token
)

#Init
Diaro_UID = "12345678901234567801234567890123"
Lat = {}
Lng = {}
UID = {} 
TAGS = {}
Lat[""] = ""
Lng[""] = ""

payload = {
    "id":Diaro_UID,
    "title":"Diaro Import"
}

try:
    #resp = requests.post(url_folders, json=payload)
    resp = requests.post(url_folders, data=json.dumps(payload, separators=(',',':')), headers=headers)
    #time.sleep(1)
    resp.raise_for_status()
    resp_dict = resp.json()
    print(resp_dict)
    print("My ID")
    print(resp_dict['id'])
    Diaro_UID_real = resp_dict['id']
    save = str(resp_dict['id'])
    UID[Diaro_UID]= save
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

print("Start : Parse Table")
tree = etree.parse("./DiaroBackup.xml")
for table in tree.iter('table'):
    name = table.attrib.get('name')
    print(name)
    myorder = 1
    for r in table.iter('r'):
         myuid = ""
         mytitle = ""
         mylat = ""
         mylng = ""
         mytags = ""
         mydate = ""
         mydate_ms = 0;
         mytext = ""
         myfilename = ""
         myfolder_uid = Diaro_UID
         mylocation_uid = ""
         myprimary_photo_uid = ""
         myentry_uid = ""
         myorder += 1
         nb_import += 1
         for subelem in r:
             print(subelem.tag)
             if (subelem.tag == 'uid'):
                 myuid = subelem.text
                 print ("myuid",myuid)
             if (subelem.tag == 'entry_uid'):
                 myentry_uid = subelem.text
                 print ("myentry_uid",myentry_uid)
             if (subelem.tag == 'primary_photo_uid'):
                 myprimary_photo_uid = subelem.text
                 print ("myprimary_photo_uid",myprimary_photo_uid)
             if (subelem.tag == 'folder_uid'):
                 myfolder_uid = subelem.text
                 print ("myfolder_uid",myfolder_uid)
             if (subelem.tag == 'location_uid'):
                 mylocation_uid = subelem.text
                 print ("mylocation_uid",mylocation_uid)
             if (subelem.tag == 'date'):
                 mydate = subelem.text
                 mydate_ms = int(mydate)
                 print ("mydate",mydate," in ms",mydate_ms)
             if (subelem.tag == 'title'):
                 mytitle = subelem.text
                 print ("mytitle",mytitle)
             if (subelem.tag == 'lat'):
                 mylat = subelem.text
                 print ("mylat",mylat)
             if (subelem.tag == 'lng'):
                 mylng = subelem.text
                 print ("mylng",mylng)
             if (subelem.tag == 'tags'):
                 mytags = subelem.text
                 if mytags:
                    mytags[1:]
                 print ("mytags",mytags)
             if (subelem.tag == 'text'):
                 mytext = subelem.text
                 print ("mytext",mytext)
                 #if type(mytext) == str:
                       #mytext = mytext.encode('utf8')
             if (subelem.tag == 'filename'):
                 myfilename = subelem.text
                 print ("myfilename",myfilename)
                 
         if (name == 'diaro_folders'):
            payload_folder = {
  "id":myuid,
  "title":mytitle,
  "parent_id":Diaro_UID_real
}
            print(payload_folder)
            try:
                #resp = requests.post(url_folders, json=payload_folder)
                resp = requests.post(url_folders, data=json.dumps(payload_folder,separators=(',',':')), headers=headers)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                save = str(resp_dict['id']) 
                UID[myuid]= save
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

         if (name == 'diaro_tags'):
            payload_tags = {
                "id":myuid,
                "title":mytitle
            }
            try:
                #resp = requests.post(url_tags, json=payload_tags)
                resp = requests.post(url_tags, data=json.dumps(payload_tags,separators=(',',':')), headers=headers)
                #time.sleep(1)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                UID[myuid]= resp_dict['id']
                TAGS[myuid] = mytitle
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

         if (name == 'diaro_attachments'):
            print("Push : "+myfilename)
            filename = "media/photo/" + myfilename
            print("----------0-----------")

            cmd = "curl -F 'data=@"+filename+"' -F 'props={\"title\":\""+myfilename+"\"}' http://"+ip+":"+port+"/resources?token="+token
            resp = os.popen(cmd).read()
            respj = json.loads(resp)
            print(respj['id'])
            UID[myuid]= respj['id']

            print("Link : ",myuid," => ",myentry_uid," // ",UID[myuid]+" => ",UID[myentry_uid])
            time.sleep(1)

            # Not possible : sniff !
            #cmd = "curl -X PUT http://"+ip+":"+port+"/ressources/"+UID[myuid]+"/notes/"+UID[myentry_uid]+"?token="+token
            #resp = os.popen(cmd).read()
            #print (resp)

            url_link = (
               "http://"+ip+":"+port+"/notes/"+UID[myentry_uid]+"?"
               "token="+token
               )
            try:
               resp = requests.get(url_link)
               resp.raise_for_status()
               resp_dict = resp.json()
               print(resp_dict)
               mybody= resp_dict['body']
            except requests.exceptions.HTTPError as e:
               print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
               print("Network error:", e)

            mybody = mybody + "\n  ![" + myfilename + "](:/" + UID[myuid] + ")   \n";
            payload_note = {
                "body":mybody
            }
            try:
               resp = requests.put(url_link, json=payload_note)
               resp.raise_for_status()
               resp_dict = resp.json()
               print(resp_dict)
            except requests.exceptions.HTTPError as e:
               print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
               print("Network error:", e)

         if (name == 'diaro_locations'):
              Lat[myuid] = mylat
              Lng[myuid] = mylng

         if (name == 'diaro_entries'):
            if not mytext:
                  mytext = ""
            if not myfolder_uid:
                  myfolder_uid = Diaro_UID
            if not mytags:
                  mytags = ""
            if not mylocation_uid:
                  mylocation_uid = ""
            mytext = mytext.replace("'", "")
            mytitle = mytitle.replace("'", "")
            mytext = mytext.strip("\'")
            mytitle = mytitle.strip("\'")
            mytext = mytext.strip('(')
            mytitle = mytitle.strip('(')
            listtags = mytags.split(",")
            new_tagslist = "";
            for uid_tags in listtags:
                 if (len(uid_tags) > 2):
                        if uid_tags in UID:
                             new_tagslist = new_tagslist + TAGS[uid_tags] + ",";
            print ("TAGS",mytags,"==>",new_tagslist);
            if (len(Lat[mylocation_uid]) > 2):
              payload_note = {
                "id":myuid,
                "latitude":float(Lat[mylocation_uid]),
                "longitude":float(Lng[mylocation_uid]),
                "tags":new_tagslist,
                "parent_id":UID[myfolder_uid],
                "title":mytitle,
                "source":myuid,
                "order":myorder,
                "created_time":mydate_ms,
                "user_created_time":mydate_ms,
                "user_updated_time":mydate_ms,
                "author":"Diaro",
                "body":mytext 
                }
            else:
               payload_note = {
                "id":myuid,
                "tags":new_tagslist,
                "parent_id":UID[myfolder_uid],
                "title":mytitle,
                "source":myuid,
                "order":myorder,
                "created_time":mydate_ms,
                "user_created_time":mydate_ms,
                "user_updated_time":mydate_ms,
                "author":"Diaro",
                "body":mytext
                }
            try:
                #resp = requests.post(url_notes, json=payload_note)
                resp = requests.post(url_notes, data=json.dumps(payload_note,separators=(',',':')), headers=headers)
                #time.sleep(1)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                UID[myuid]= resp_dict['id']
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

print("End : Parse Table")

strftime("%Y-%m-%d %H:%M:%S", gmtime())
done = time.time()
elapsed = done - start
print(elapsed)
print(nb_import)

# END : Ouf ...

Diaro App : DiaroBackup.xml : How to parse in python ? (Draft n°2)

93 x served & 24 x viewed

(See the finale release : http://www.cyber-neurones.org/2019/02/diaro-app-pixel-crater-ltd-diarobackup-xml-how-to-migrate-data-to-joplin/ )

I have issue with ressources (link between ressources and notes) …. error 404. The logs in : .config/joplin-desktop/log-clipper.txt

....: "Request: PUT /ressources/71dd2cba2af54c4ebb53fb7fd8d0543b/notes/cbbc6076b2ac321ccae1f036a2fe6659?token=...."
....: "Error: Not Found
Error: Not Found
    at Api.route (/Applications/Joplin.app/Contents/Resources/app/lib/services/rest/Api.js:103:41)
    at execRequest (/Applications/Joplin.app/Contents/Resources/app/lib/ClipperServer.js:147:39)
    at IncomingMessage.request.on (/Applications/Joplin.app/Contents/Resources/app/lib/ClipperServer.js:185:8)
    at emitNone (events.js:105:13)
    at IncomingMessage.emit (events.js:207:7)
    at endReadableNT (_stream_readable.js:1045:12)
    at _combinedTickCallback (internal/process/next_tick.js:138:11)
    at process._tickCallback (internal/process/next_tick.js:180:9)"

My last code :

#
# Version 2 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

#from lxml import etree
import xml.etree.ElementTree as etree
from time import gmtime, strftime
import time
import json
import requests
import os

strftime("%Y-%m-%d %H:%M:%S", gmtime())
start = time.time()

#Token
ip = "127.0.0.1"
port = "41184"
token = "ABCD123ABCD123ABCD123ABCD123ABCD123"

url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
url_folders = (
    "http://"+ip+":"+port+"/folders?"
    "token="+token
)
url_tags = (
    "http://"+ip+":"+port+"/tags?"
    "token="+token
)
url_ressources = (
    "http://"+ip+":"+port+"/ressources?"
    "token="+token
)

#Init
Diaro_UID = "12345678901234567801234567890123"
Lat = {}
Lng = {}
UID = {} 
TAGS = {}
Lat[""] = ""
Lng[""] = ""

payload = {
    "id": Diaro_UID,
    "title": "Diaro Import"
}

try:
    resp = requests.post(url_folders, json=payload)
    #time.sleep(1)
    resp.raise_for_status()
    resp_dict = resp.json()
    print(resp_dict)
    print("My ID")
    print(resp_dict['id'])
    Diaro_UID_real = resp_dict['id']
    save = str(resp_dict['id'])
    UID[Diaro_UID]= save
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

print("Start : Parse Table")
tree = etree.parse("./DiaroBackup.xml")
for table in tree.iter('table'):
    name = table.attrib.get('name')
    print(name)
    myorder = 1
    for r in table.iter('r'):
         myuid = ""
         mytitle = ""
         mylat = ""
         mylng = ""
         mytags = ""
         mydate = ""
         mydate_ms = 0;
         mytext = ""
         myfilename = ""
         myfolder_uid = Diaro_UID
         mylocation_uid = ""
         myprimary_photo_uid = ""
         myentry_uid = ""
         myorder += 1
         for subelem in r:
             print(subelem.tag)
             if (subelem.tag == 'uid'):
                 myuid = subelem.text
                 print ("myuid",myuid)
             if (subelem.tag == 'entry_uid'):
                 myentry_uid = subelem.text
                 print ("myentry_uid",myentry_uid)
             if (subelem.tag == 'primary_photo_uid'):
                 myprimary_photo_uid = subelem.text
                 print ("myprimary_photo_uid",myprimary_photo_uid)
             if (subelem.tag == 'folder_uid'):
                 myfolder_uid = subelem.text
                 print ("myfolder_uid",myfolder_uid)
             if (subelem.tag == 'location_uid'):
                 mylocation_uid = subelem.text
                 print ("mylocation_uid",mylocation_uid)
             if (subelem.tag == 'date'):
                 mydate = subelem.text
                 mydate_ms = int(mydate)
                 print ("mydate",mydate," in ms",mydate_ms)
             if (subelem.tag == 'title'):
                 mytitle = subelem.text
                 print ("mytitle",mytitle)
                 #if type(mytitle) == str:
                    #mytitle = mytitle.encode('utf8')
             if (subelem.tag == 'lat'):
                 mylat = subelem.text
                 print ("mylat",mylat)
             if (subelem.tag == 'lng'):
                 mylng = subelem.text
                 print ("mylng",mylng)
             if (subelem.tag == 'tags'):
                 mytags = subelem.text
                 if mytags:
                    mytags[1:]
                 print ("mytags",mytags)
             if (subelem.tag == 'text'):
                 mytext = subelem.text
                 print ("mytext",mytext)
                 #if type(mytext) == str:
                       #mytext = mytext.encode('utf8')
             if (subelem.tag == 'filename'):
                 myfilename = subelem.text
                 print ("myfilename",myfilename)
                 
         if (name == 'diaro_folders'):
            payload_folder = {
  "id": myuid,
  "title": mytitle,
  "parent_id": Diaro_UID_real
}
            print(payload_folder)
            try:
                resp = requests.post(url_folders, json=payload_folder)
                #time.sleep(1)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                save = str(resp_dict['id']) 
                UID[myuid]= save
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

         if (name == 'diaro_tags'):
            payload_tags = {
                "id": myuid,
                "title": mytitle
            }
            try:
                resp = requests.post(url_tags, json=payload_tags)
                #time.sleep(1)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                UID[myuid]= resp_dict['id']
                TAGS[myuid] = mytitle
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

         if (name == 'diaro_attachments'):
            payload_ressource = {
                "id": myuid
            }
            filename = "./media/photo/"+myfilename
            files = {'document': open(filename, 'rb')}
            files2 = {'data': open(filename, 'rb')}
            files3 = {'data': open(filename, 'rb'), 'props': payload_ressource}
            data_ressource = {
                 "title": myfilename
            }
            multiple_files = [
                ('data', (myfilename, open(filename, 'rb'))),
                ('props', data_ressource)]
            headers = {'Content-type': 'multipart/form-data'}
            print("Push : "+filename);
            #print os.path.isfile(filename)
            print("----------0-----------")
            #try:
               #resp = requests.post(url_ressources, files=filename, json=payload_ressource)
               #resp = requests.post(url_ressources, files=files, json=payload_ressource, headers=headers) 
               #resp = requests.post(url_ressources, files=files2, headers=headers)
               #resp = requests.post(url_ressources, files=files2, headers=headers)
               #resp = requests.post(url_ressources,files = {'data' : (myfilename, open(filename, 'rb'), 'image/jpg')}, data = {'id' : myuid}, headers=headers)
               #resp = requests.post(url_ressources,files = files2, data= data_ressource, headers=headers)
               #resp = requests.post(url_ressources,files = multiple_files, headers=headers)
               #resp = requests.post(url_ressources,files = multiple_files)
               #resp.text
               #time.sleep(1)
               #resp.raise_for_status()
               #if (resp.status_code == requests.codes.ok):
               #    resp_dict = resp.json()
               #    print(resp_dict)
               #    print(resp_dict['id'])
               #    UID[myuid]= resp_dict['id']
            #except requests.exceptions.HTTPError as e:
               #print("Bad HTTP status code:", e)
               #UID[myuid]=""
               #print("----------1-----------")
            #except requests.exceptions.RequestException as e:
               #print("Network error:", e)
               #UID[myuid]=""
               #print("----------2-----------")

            cmd = "curl -F 'data=@"+filename+"' -F 'props={\"title\":\""+myfilename+"\"}' http://"+ip+":"+port+"/resources?token="+token
            resp = os.popen(cmd).read()
            respj = json.loads(resp)
            #resp_dict = respj.json() 
            print(respj['id'])
            UID[myuid]= respj['id']

            print("Link : ",myuid," => ",myentry_uid," // ",UID[myuid]+" => ",UID[myentry_uid])
            time.sleep(1)

            cmd = "curl -X PUT http://"+ip+":"+port+"/ressources/"+UID[myuid]+"/notes/"+UID[myentry_uid]+"?token="+token
            resp = os.popen(cmd).read()
            print (resp)
            #url_link = (
            #   "http://"+ip+":"+port+"/ressources/"+UID[myuid]+"/notes/"+UID[myentry_uid]+"?"
            #   "token="+token
            #   )
            #try:
            #  resp = requests.post(url_link)
            #   #time.sleep(1)
            #   resp.raise_for_status()
            #   resp_dict = resp.json()
            #   print(resp_dict)
            #   print(resp_dict['id'])
            #   UID[myuid]= resp_dict['id']
            #except requests.exceptions.HTTPError as e:
            #   print("Bad HTTP status code:", e)
            #except requests.exceptions.RequestException as e:
            #   print("Network error:", e)

         if (name == 'diaro_locations'):
              Lat[myuid] = mylat
              Lng[myuid] = mylng

         if (name == 'diaro_entries'):
            if not mytext:
                  mytext = ""
            if not myfolder_uid:
                  myfolder_uid = Diaro_UID
            if not mytags:
                  mytags = ""
            if not mylocation_uid:
                  mylocation_uid = ""
            mytext = mytext.replace("'", "")
            mytitle = mytitle.replace("'", "")
            mytext = mytext.strip("\'")
            mytitle = mytitle.strip("\'")
            mytext = mytext.strip('(')
            mytitle = mytitle.strip('(')
            listtags = mytags.split(",")
            new_tagslist = "";
            for uid_tags in listtags:
                 if (len(uid_tags) > 2):
                        if uid_tags in UID:
                             new_tagslist = new_tagslist + TAGS[uid_tags] + ",";
            print ("TAGS",mytags,"==>",new_tagslist);
            payload_note = {
                "id": myuid,
                "latitude": Lat[mylocation_uid],
                "longitude": Lng[mylocation_uid],
                "tags": new_tagslist,
                "parent_id": UID[myfolder_uid],
                "title": mytitle,
                #"created_time": mydate_ms,
                "user_created_time": mydate_ms,
                "user_updated_time": mydate_ms,
                "author": "Diaro",
                "body": mytext 
            }
            try:
                resp = requests.post(url_notes, json=payload_note)
                #time.sleep(1)
                resp.raise_for_status()
                resp_dict = resp.json()
                print(resp_dict)
                print(resp_dict['id'])
                UID[myuid]= resp_dict['id']
            except requests.exceptions.HTTPError as e:
                print("Bad HTTP status code:", e)
            except requests.exceptions.RequestException as e:
                print("Network error:", e)

print("End : Parse Table")

strftime("%Y-%m-%d %H:%M:%S", gmtime())
done = time.time()
elapsed = done - start
print(elapsed)

# END : Ouf ...

Mac OS : ImportError: No module named requests

168 x served & 52 x viewed

J’ai voulu suivre la procédure avec brew, pip, …. mais sans succès avec la version 2.7.2

$ python --version
Python 2.7.2

J’avais des erreurs du type :

$ brew reinstall python
....
xcrun: error: invalid active developer path (/Library/Developer/CommandLineTools), missing xcrun at: /Library/Developer/CommandLineTools/usr/bin/xcrun
Error: An exception occurred within a child process:
  CompilerSelectionError: python cannot be built with any available compilers.
Install GNU's GCC
  brew install gcc

$ python -m pip install --user requests
/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python: No module named pip

$ sudo easy_install pip
Searching for pip
Reading http://pypi.python.org/simple/pip/
Couldn't find index page for 'pip' (maybe misspelled?)
Scanning index of all packages (this may take a while)
Reading http://pypi.python.org/simple/
No local packages or download links found for pip
Best match: None
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/2.7/bin/easy_install", line 8, in <module>
    load_entry_point('setuptools==0.6c11', 'console_scripts', 'easy_install')()
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/command/easy_install.py", line 1712, in main
    
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/command/easy_install.py", line 1700, in with_ei_usage
    
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/command/easy_install.py", line 1716, in <lambda>
    
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/core.py", line 152, in setup
    dist.run_commands()
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/dist.py", line 953, in run_commands
    self.run_command(cmd)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/distutils/dist.py", line 972, in run_command
    cmd_obj.run()
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/command/easy_install.py", line 211, in run
    
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/command/easy_install.py", line 434, in easy_install
    
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/setuptools-0.6c11-py2.7.egg/setuptools/package_index.py", line 475, in fetch_distribution
AttributeError: 'NoneType' object has no attribute 'clone'



J’ai donc changé de fusil d’épaule :

$ brew install python3
Warning: Building python from source:
  The bottle needs the Apple Command Line Tools to be installed.
  You can install them, if desired, with:
    xcode-select --install

xcrun: error: invalid active developer path (/Library/Developer/CommandLineTools), missing xcrun at: /Library/Developer/CommandLineTools/usr/bin/xcrun
Error: An exception occurred within a child process:
  CompilerSelectionError: python cannot be built with any available compilers.
Install GNU's GCC
  brew install gcc

$ xcode-select --install
...
$ brew install python3
...
$ brew link --overwrite python
Linking /usr/local/Cellar/python/3.7.2_1... Error: Permission denied @ dir_s_mkdir - /usr/local/Frameworks

$ sudo mkdir /usr/local/Frameworks
...
$ sudo chown $(whoami):admin /usr/local/Frameworks
...
$ brew link --overwrite python
Linking /usr/local/Cellar/python/3.7.2_1... 1 symlinks created
$ python --version
Python 2.7.2
$ ls -l /usr/bin/py*
-rwxr-xr-x  4 root  wheel    925 18 aoû 02:45 /usr/bin/pydoc
lrwxr-xr-x  1 root  wheel     74 24 jan 08:32 /usr/bin/pydoc2.7 -> ../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/pydoc2.7
-rwxr-xr-x  1 root  wheel  66880 16 jan 02:21 /usr/bin/python
-rwxr-xr-x  4 root  wheel    925 18 aoû 02:45 /usr/bin/python-config
lrwxr-xr-x  1 root  wheel     75 24 jan 08:32 /usr/bin/python2.7 -> ../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
lrwxr-xr-x  1 root  wheel     82 24 jan 08:32 /usr/bin/python2.7-config -> ../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7-config
-rwxr-xr-x  1 root  wheel  66880 16 jan 02:21 /usr/bin/pythonw
lrwxr-xr-x  1 root  wheel     76 24 jan 08:32 /usr/bin/pythonw2.7 -> ../../System/Library/Frameworks/Python.framework/Versions/2.7/bin/pythonw2.7
....
$ brew reinstall python
...
$ pip3 install requests
...

Et cela fonctionne … en résumé, le Python c’est super simple a installer. Enfin au bout d’un moment cela tombe en marche et tu es bien content …. Misère.

Tout cela pour écrire quelques lignes de Python ( Pour utiliser l’API REST JOLIN en Python : https://joplin.cozic.net/api/ ) :

import json
import requests

ip = "127.0.0.1"
port = "41184"
Diaro_UID = "12345678901234567801234567890123"
token = "bcf7475af70806df3e0b88dd10fcf88971e7088e31"
url = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
url2 = (
    "http://"+ip+":"+port+"/folders?"
    "token="+token
)
payload = {
    "id": Diaro_UID,
    "title": "Diaro Import"
}

try:
    resp = requests.post(url2, json=payload)
    resp.raise_for_status()
    resp_dict = resp.json()
    print(resp_dict)
    print("My ID")
    print(resp_dict['id'])
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

Diaro App : DiaroBackup.xml : How to parse in python ? (Draft n°1)

236 x served & 33 x viewed

(See the finale release : http://www.cyber-neurones.org/2019/02/diaro-app-pixel-crater-ltd-diarobackup-xml-how-to-migrate-data-to-joplin/ )

Step 1: Add in first ligne : <?xml version= »1.0″ encoding= »UTF-8″?> before <data> in file DiaroBackup.xml … it’s mandatory !

I use REST API to insert in JOPLIN : https://joplin.cozic.net/api/ , it’s good documentation.

Here my first release in Python to import data from Diaro App Backup to Joplin API :

#
# Version 1  
#  
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)

from urllib2 import unquote
from lxml import etree
import os
from time import gmtime, strftime
import time

strftime("%Y-%m-%d %H:%M:%S", gmtime())
start = time.time()

print("Start : Parse Table")
tree = etree.parse("./DiaroBackup.xml")
for table in tree.xpath("/data/table"):
    print(table.get("name"))
print("End : Parse Table")

#Token
ip = "127.0.0.1"
port = "41184"
#token = "ABCD123ABCD123ABCD123ABCD123ABCD123"
token = "blablabla"
cmd = 'curl http://'+ip+':'+port+'/notes?token='+token
print cmd
os.system(cmd)

#Init
Diaro_UID = "12345678901234567801234567890123"
Lat = {}
Lng = {}
Lat[""] = ""
Lng[""] = ""
cmd = 'curl --data \'{ "id": "'+Diaro_UID+'", "title": "Diaro Import"}\' http://'+ip+':'+port+'/folders?token='+token
print cmd
os.system(cmd)

print("Start : Parse Table")
tree = etree.parse("./DiaroBackup.xml")
for table in tree.iter('table'):
    name = table.attrib.get('name')
    print name
    myorder = 1
    for r in table.iter('r'):
         myuid = ""
         mytitle = ""
         mylat = ""
         mylng = ""
         mytags = ""
         mydate = ""
         mytext = ""
         myfilename = ""
         myfolder_uid = Diaro_UID
         mylocation_uid = ""
         myprimary_photo_uid = ""
         myentry_uid = ""
         myorder += 1
         for subelem in r:
	     print(subelem.tag)
             if (subelem.tag == 'uid'):
                 myuid = subelem.text
              	 print ("myuid",myuid)
             if (subelem.tag == 'entry_uid'):
                 myentry_uid = subelem.text
                 print ("myentry_uid",myentry_uid)
             if (subelem.tag == 'primary_photo_uid'):
                 myprimary_photo_uid = subelem.text
                 print ("myprimary_photo_uid",myprimary_photo_uid)
             if (subelem.tag == 'folder_uid'):
                 myfolder_uid = subelem.text
                 print ("myfolder_uid",myfolder_uid)
             if (subelem.tag == 'location_uid'):
                 mylocation_uid = subelem.text
                 print ("mylocation_uid",mylocation_uid)
             if (subelem.tag == 'date'):
                 mydate = subelem.text
                 print ("mydate",mydate)
             if (subelem.tag == 'title'):
                 mytitle = subelem.text
                 print ("mytitle",mytitle)
		 print type(mytitle)
                 if type(mytitle) == unicode:
			mytitle = mytitle.encode('utf8')
             if (subelem.tag == 'lat'):
                 mylat = subelem.text
                 print ("mylat",mylat)
             if (subelem.tag == 'lng'):
                 mylng = subelem.text
                 print ("mylng",mylng)
             if (subelem.tag == 'tags'):
                 mytags = subelem.text
                 if mytags:
                    mytags[1:]
                 print ("mytags",mytags)
             if (subelem.tag == 'text'):
                 mytext = subelem.text
                 print ("mytext",mytext)
                 if type(mytext) == unicode:
                        mytext = mytext.encode('utf8')
             if (subelem.tag == 'filename'):
                 myfilename = subelem.text
                 print ("myfilename",myfilename)
         if (name == 'diaro_folders'):
              cmd = 'curl --data \'{ "id": "'+myuid+'", "title": "'+mytitle+'", "parent_id": "'+Diaro_UID+'"}\' http://'+ip+':'+port+'/folders?token='+token
              print cmd
              os.system(cmd)
         if (name == 'diaro_tags'):
              cmd = 'curl --data \'{ "id": "'+myuid+'", "title": "'+mytitle+'"}\' http://'+ip+':'+port+'/tags?token='+token
              print cmd
              os.system(cmd)
         if (name == 'diaro_attachments'):
              cmd = 'curl -F \'data=@media/photo/'+myfilename+'\'  -F \'props={"id":"'+myuid+'"}\' http://'+ip+':'+port+'/resources?token='+token
              print cmd
              os.system(cmd)
              cmd = 'curl -X PUT http://'+ip+':'+port+'/resources/'+myuid+'/notes/'+myentry_uid+'?token='+token
              print cmd
              os.system(cmd)
         if (name == 'diaro_locations'):
              Lat[myuid] = mylat
              Lng[myuid] = mylng
         if (name == 'diaro_entries'):
             if not mytext:
                  mytext = ""
             if not myfolder_uid:
                  myfolder_uid = Diaro_UID
             if not mytags:
                  mytags = ""
             if not mylocation_uid:
                  mylocation_uid = ""
             mytext = mytext.replace("'", "")
             mytitle = mytitle.replace("'", "")
             mytext = mytext.strip("\'")
             mytitle = mytitle.strip("\'")
             mytext = mytext.strip('(')
             mytitle = mytitle.strip('(')
             print type(mytext)
             cmd = 'curl --data \'{"latitude":"'+Lat[mylocation_uid]+'","longitude":"'+Lng[mylocation_uid]+'","tags":"'+mytags+'","parent_id":"'+myfolder_uid+'","id":"'+myuid+'","title":"'+mytitle+'", "created_time": "'+mydate+'", "body": "'+mytext+'"}\' http://'+ip+':'+port+'/notes?token='+token
             print cmd
             os.system(cmd)
print("End : Parse Table")

strftime("%Y-%m-%d %H:%M:%S", gmtime())
done = time.time()
elapsed = done - start
print(elapsed)

But I don’t understand the API, I can force the id ( for exemple : 12345678901234567801234567890123 ):

curl --data '{ "id": "12345678901234567801234567890123", "title": "Diaro Import"}' http://127.0.0.1:41184/folders?token=.....
{"title":"Diaro Import","id":"73d15fe0b55e40dabea353b0f9d45547","updated_time":1549406274867,"created_time":1549406274867,"user_updated_time":1549406274867,"user_created_time":1549406274867,"type_":2}

Same issue with ressource :

curl -F 'data=@media/photo/photo_20181202_810728.jpg'  -F 'props={"id":"fe2a86a78bded44f18d6da73fff9a3e1"}' http://127.0.0.1:41184/resources?token=.....
{"id":"68d662d3b7f0421f9dc4bf8f33bdc74c","title":"UpPr0WId4ZsIFiLAAL6ZLMf1.jpg","mime":"image/jpeg","filename":"","created_time":1549409257224,"updated_time":1549409257229,"user_created_time":1549409257224,"user_updated_time":1549409257229,"file_extension":"jpg","encryption_cipher_text":"","encryption_applied":0,"encryption_blob_encrypted":0,"type_":4}

And also all my notes are tags ?!

Logs of Joplin ( .config/joplin-desktop/log.txt ) , before import :

2019-02-05 23:25:51: "Running background sync on timer..."
2019-02-05 23:25:51: "Scheduling sync operation..."
2019-02-05 23:25:51: "Preparing scheduled sync"
2019-02-05 23:25:51: "Starting scheduled sync"
2019-02-05 23:25:51: "Operations completed: "
2019-02-05 23:25:51: "fetchingTotal: -"
2019-02-05 23:25:51: "Total folders: 8"
2019-02-05 23:25:51: "Total notes: 23"
2019-02-05 23:25:51: "Total resources: 35"

Logs of Joplin, after import (no information on Tags):

2019-02-06 00:28:49: "Scheduling sync operation..."
2019-02-06 00:28:49: "Preparing scheduled sync"
2019-02-06 00:28:49: "Starting scheduled sync"
2019-02-06 00:28:49: "Operations completed: "
2019-02-06 00:28:49: "createRemote: 2"
2019-02-06 00:28:49: "fetchingTotal: 2"
2019-02-06 00:28:49: "Total folders: 24"
2019-02-06 00:28:49: "Total notes: 460"
2019-02-06 00:28:49: "Total resources: 891"

But I don’t see any notes … but the note are on tags ?!

How to start the API :

And when you active the service, you see the port :

And not easy to contact the forum : https://discourse.joplin.cozic.net/t/diario-awesome-notes-webdavnav/1518/6