Python : How to migrate data of Awesome Note 2 (bridworks.com) to Joplin ?

Awesome Note 2, it’s very popular on iPad :

The new All-in-one Organizer, Awesome Note 2 is integrated with note and schedule management.
And now it’s available!!

WONDERFUL WRITING FEATURES
· It can be used not only for simple notes, but also rich and wonderful writing tool.
· Make notes even more powerful to add photos, voice recording and drawings.
· Easily create diary notes to display feeling, weather or road map information.

SIMPLE, FLEXIBLE, AND FRIENDLY
·  Broadly use as diary to record everyday life, travel notes to write anywhere, photo albums, shopping lists, and record for work or study in any theme.
 
NOTE AND SCHEDULING AS ONE
· Manage your iOS Calendar and Reminders in one.
· Check todo lists and manage all schedules with calendar
· Receive notifications for important events and easily manage anniversaries such as birthdays.
 
NEAT AND STYLISH DESIGN
· Create your own style with tastefully designed icons, folders and various note backgrounds.
 
Capture all the memorable moments, stories, and everything in you. 

Step 0 : Install Joplin and activate the REST API ( https://joplin.cozic.net/api/ ) .

Step 1 : Install Python.

Step 2 : Create a backup of Awesome Note. ( for exemple : aNote_13Folders_20170520_00_24_21_579Notes.anb )

Step 3 : Uncompress the backup (It’s zip).

Step 4 : Put the script and change Token and name of folder of Backup.

The script :

#
# Version 1 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

import plistlib
import os
import glob

folder = "Put the name of folder"

import requests
import re
import json
from subprocess import Popen, PIPE
import xml.etree.ElementTree
import xml.etree.ElementTree as ET
from xml.dom import minidom
from bpylist import bplist
import base64

###

#IP
ip = "127.0.0.1"
#Port
port = "41184"
#Token
token = "Put the token"
nb_import = 0;
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}

url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
url_folders = (
    "http://"+ip+":"+port+"/folders?"
    "token="+token
)
url_tags = (
    "http://"+ip+":"+port+"/tags?"
    "token="+token
)
url_ressources = (
    "http://"+ip+":"+port+"/ressources?"
    "token="+token
)

#Init
Awesome_UID = "12345678901234567801234567890123"
Awesome_UID_real = ""

payload = {
    "id":Awesome_UID,
    "title":"Awesome Note Import"
}

try:
    resp = requests.post(url_folders, data=json.dumps(payload, separators=(',',':')), headers=headers)
    #time.sleep(1)
    resp.raise_for_status()
    resp_dict = resp.json()
    print(resp_dict)
    print("My ID")
    print(resp_dict['id'])
    Awesome_UID_real = resp_dict['id']
    save = str(resp_dict['id'])
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

###
nb_picture = 0
order = 0

# Convert to XML
files = os.listdir(folder)
for name in files:
  if name.endswith('.anote'):
    #print(name)
    name2 = os.path.splitext(name)[0]
    my_file = folder+"/"+name2+".xml"
    name = name.replace(" ", "\\ ")
    name = name.replace("(", "\\(")
    name = name.replace(")", "\\)")
    name = name.replace("&", "\\&")
    name2 = name2.replace(" ", "\\ ")
    name2 = name2.replace("(", "\\(")
    name2 = name2.replace(")", "\\)")
    name2 = name2.replace("&", "\\&")
    #print (name2)
    commande = "plutil -convert xml1 "+folder+"/"+name+" -o "+folder+"/"+name2+".xml"
    os.system(commande)
    my_dict = {}
    nb_array = 0
    nb_integer = 0
    nb_real = 0
    nb_string = 0
    flag_ok = 0
    nb_data = 0
    liste_picture = []
    my_timestamp = 0
    my_title = ""
    my_body = ""
    my_id = ""
    my_picture_name = ""
    my_lat = ""
    my_long = ""
    my_address = ""
    for event, elem in ET.iterparse(my_file):
        assert event == 'end'
        if elem.tag == 'array':
            key = elem.text
            nb_array += 1;
        elif elem.tag == 'key':
            value = elem.text
            if (value == 'Version'):
               flag_ok = 1
               #print("ok")
               nb_array = 0
               nb_integer = 0
               nb_real = 0
               nb_string = 0
               nb_data = 0
        elif elem.tag == 'integer':
            nb_integer += 1
            value = elem.text
        elif elem.tag == 'real':
            nb_real += 1
            value = elem.text
            if (flag_ok == 1):
                if (nb_real == 1):
                   my_timestamp = float(value)*1000;
        elif elem.tag == 'string':
            nb_string += 1
            value = elem.text
            if (flag_ok == 1):
                if (nb_string == 2):
                   my_key = value
                   if (value is None):
                      nb_string += 0
                   else:
                      if (value.startswith("Apple")):
                         nb_string += 1
                      elif (value.startswith("Times")):
                         nb_string += 1
                      else:
                         split_list = value.split()
                         if len(split_list) == 3:
                              my_long, my_lat, my_address = value.split("|")
                              print("Address",my_long, my_lat, my_address)
                if (nb_string == 4):
                   my_key = value
                   if (value is None):
                      nb_string += 0
                   else:
                      if (value.startswith("Apple")):
                         nb_string -= 1
                      elif (value.startswith("Times")):
                         nb_string -= 1
                if (nb_string == 5):
                   my_key = value
                   if (value is None):
                      nb_string += 0
                   else:
                      if (value.startswith("Apple")):
                         nb_string -= 2 
                      elif (value.startswith("Times")):
                         nb_string -= 2 
                if (nb_string == 5):
                   my_body = value
                   if (my_body is None):
                       my_body = ""
                   else:
                       my_body = re.sub(r"<@b>", "**", my_body)
                       my_body = re.sub(r"</@b>", "**", my_body)
                       my_body = re.sub(r"<@u>", "*", my_body)
                       my_body = re.sub(r"</@u>", "*", my_body)
                if (nb_string == 6):
                   my_title = value
                   if (my_title is None):
                       my_title = ""
                   elif (len(my_title) == 36 and (' ' not in my_title)):
                       if (len(my_body) > 0):
                           my_title = my_body
                if (nb_string == 7):
                   my_id = value
                   my_id = my_id.replace("-", "")
                   my_id = my_id[0:31]
        elif elem.tag == 'entry':
            my_dict[key] = value
            key = value = None
        elif elem.tag == 'data':
            nb_data += 1
            value = elem.text
            value_clean = re.sub(r" ", "", value)
            value_clean = re.sub(r"\t", "", value_clean)
            value_clean = re.sub(r"\n", "", value_clean)
            nb_picture += 1
            my_picture_name = str(nb_picture)+".jpg"
            liste_picture.append(my_picture_name)
            jpg_recovered = base64.decodestring(value_clean.encode())
            g = open(my_picture_name, "wb")
            g.write(jpg_recovered)
            g.close()
        elem.clear()

    #print("Data:",my_timestamp,",",my_title,",",my_body)
    if (len(my_title) > 150):
       print("Error",my_file);
    print("Filename:",my_file,"Data:",my_timestamp,",",my_title,",",my_id)
    nb_import += 1
    payload_note = {
                "parent_id":Awesome_UID_real,
                "title":my_title,
                "source":my_file,
                "order":nb_import,
                "user_created_time":my_timestamp,
                "user_updated_time":my_timestamp,
                "author":"Awesome Note",
                "body":my_body
}
    if (len(my_address) > 0):
        payload_note_put = {
                "source":my_file,
                "longitude":float(my_long),
                "latitude":float(my_lat),
                "order":nb_import,
                "user_created_time":my_timestamp,
                "user_updated_time":my_timestamp,
                "author":"Awesome Note"
}
    else :
        payload_note_put = {
                "source":my_file,
                "order":nb_import,
                "user_created_time":my_timestamp,
                "user_updated_time":my_timestamp,
                "author":"Awesome Note"
}

    myuid = my_id

    try:
        resp = requests.post(url_notes, json=payload_note)
        resp.raise_for_status()
        resp_dict = resp.json()
        print(resp_dict)
        myuid= resp_dict['id']
        my_id = myuid
    except requests.exceptions.HTTPError as e:
        print("Bad HTTP status code:", e)
        print("payload_note:", payload_note)
    except requests.exceptions.RequestException as e:
        print("Network error:", e)

    url_notes_put = (
    "http://"+ip+":"+port+"/notes/"+myuid+"?"
    "token="+token
)

    try:
        resp = requests.put(url_notes_put, json=payload_note_put)
        resp.raise_for_status()
        resp_dict = resp.json()
        print(resp_dict)
    except requests.exceptions.HTTPError as e:
        print("Bad HTTP status code:", e)
        print("payload_note:", payload_note_put)
    except requests.exceptions.RequestException as e:
        print("Network error:", e)

    for my_picture_name in liste_picture:
               cmd = "curl -F 'data=@"+my_picture_name+"' -F 'props={\"title\":\""+my_picture_name+"\"}' http://"+ip+":"+port+"/resources?token="+token
               print("Command"+cmd)
               resp = os.popen(cmd).read()
               try:
                  respj = json.loads(resp)
                  print(respj['id'])
                  myuid_picture= respj['id']
               except:
                  print('bad json: ', resp)

               my_body = my_body + "\n  ![" + my_picture_name + "](:/" + myuid_picture + ")   \n";

               payload_note_put = {
                "body":my_body
                }

               try:
                  resp = requests.put(url_notes_put, json=payload_note_put)
                  resp.raise_for_status()
                  resp_dict = resp.json()
                  print(resp_dict)
               except requests.exceptions.HTTPError as e:
                  print("Bad HTTP status code:", e)
                  print("payload_note:", payload_note_put)
               except requests.exceptions.RequestException as e:
                  print("Network error:", e)


With this script you have : Title, Body, Picture, Location. ( No tags but it’s possible, and no folder ). It’s impossible to use plistlib …. sniff.


Python : How to do a Word Cloud with data (Title or body) in Joplin ?

Step 0 : Install Joplin and activate the REST API ( https://joplin.cozic.net/api/ ) .

Step 1: Install nltk and worldcloud with pip ( for more information see https://www.datacamp.com/community/tutorials/wordcloud-python )

Step 3.a : Run this scripts for Title (change the token)

#
# Version 1 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

from time import gmtime, strftime
import time
import json
import requests
import os
import nltk
nltk.download('punkt')
nltk.download('stopwords')
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from wordcloud import WordCloud
import numpy as np
import matplotlib.pyplot as plt

#IP
ip = "127.0.0.1"
#Port
port = "41184"
#Token
token = "Put your token here"
nb_request = 0
my_title = ""
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
try:
    resp = requests.get(url_notes, headers=headers)
    nb_request += 1
    resp.raise_for_status()
    resp_dict = resp.json()
    #print(resp_dict)
    for my_note in resp_dict:
        #print(my_note.get('id'))
        my_title += my_note.get('title')
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

# Create a word cloud image
stopwords = stopwords.words('french')
wc = WordCloud(background_color="white", max_words=5000, stopwords=stopwords, contour_width=3, contour_color='firebrick')
wc.generate(my_title)
wc.to_file("jopling_title.png")
plt.figure(figsize=[18,8])
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()

Step 3.b : Run this scripts for Body (change the token)

#
# Version 1 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

from time import gmtime, strftime
import time
import json
import requests
import os
import nltk
nltk.download('punkt')
nltk.download('stopwords')
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from wordcloud import WordCloud
import numpy as np
import matplotlib.pyplot as plt

#IP
ip = "127.0.0.1"
#Port
port = "41184"
#Token
token = "Put your token here"
nb_request = 0
my_body = ""
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
try:
    resp = requests.get(url_notes, headers=headers)
    nb_request += 1
    resp.raise_for_status()
    resp_dict = resp.json()
    #print(resp_dict)
    for my_note in resp_dict:
        #print(my_note.get('id'))
        my_body += my_note.get('body')
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

# Create a word cloud image
stopwords = stopwords.words('french')
wc = WordCloud(background_color="white", max_words=5000, stopwords=stopwords, contour_width=3, contour_color='firebrick')
wc.generate(my_body)
wc.to_file("jopling_body.png")
plt.figure(figsize=[18,8])
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()

Off course it’s possible to change the maximum of words with : max_words=5000 . And also the size with figsize=[18,8].

Joplin & Python ( Static Map ) : How to create a Maps (JPEG) with REST API and data in Joplin ?

Step 0 : Install Joplin and activate the REST API ( https://joplin.cozic.net/api/ ) .

Step 1: Install staticmap with pip ( for more information see https://github.com/komoot/staticmap )

$ pip install staticmap
Collecting staticmap
  Downloading https://files.pythonhosted.org/packages/f9/9f/5a3843533eab037cba031486175c4db1b214614404a29516208ff228dead/staticmap-0.5.4.tar.gz
Collecting Pillow (from staticmap)
  Downloading https://files.pythonhosted.org/packages/c9/ed/27cc92e99b9ccaa0985a66133baeea7e8a3371d3c04cfa353aaa3b81aac1/Pillow-5.4.1-cp37-cp37m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl (3.7MB)
    100% |████████████████████████████████| 3.7MB 6.3MB/s 
Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (from staticmap) (2.21.0)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/site-packages (from requests->staticmap) (3.0.4)
Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests->staticmap) (2.8)
Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests->staticmap) (1.24.1)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests->staticmap) (2018.11.29)
Building wheels for collected packages: staticmap
  Building wheel for staticmap (setup.py) ... done
  Stored in directory: /Users/..../Library/Caches/pip/wheels/fe/a6/a5/2acceb72471d85bd0498973aabd611e6ff1cdd48796790f047
Successfully built staticmap
Installing collected packages: Pillow, staticmap
Successfully installed Pillow-5.4.1 staticmap-0.5.4

The source code :

#
# Version 1
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

from time import gmtime, strftime
import time
import json
import requests
import os
import gmplot
from staticmap import StaticMap, CircleMarker

m = StaticMap(1600, 800, url_template='http://a.tile.osm.org/{z}/{x}/{y}.png')

marker = CircleMarker((10, 47), '#0036FF', 3)
m.add_marker(marker)

#Token
ip = "127.0.0.1"
port = "41184"

token = "Put your token here"
nb_request = 0
nb_plot = 0
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
try:
    resp = requests.get(url_notes, headers=headers)
    nb_request += 1
    resp.raise_for_status()
    resp_dict = resp.json()
    #print(resp_dict)
    for my_note in resp_dict:
        #print(my_note.get('id'))
        url_notes2 = (
    "http://"+ip+":"+port+"/notes/"+my_note.get('id')+"?fields=longitude,latitude&"
    "token="+token
)
        try:
           resp2 = requests.get(url_notes2, headers=headers)
           nb_request += 1
           resp2.raise_for_status()
           resp_dict2 = resp2.json()
           #print(resp_dict2)
           long = resp_dict2.get('longitude')
           lat = resp_dict2.get('latitude')
           if (long != '0.00000000'):
              nb_plot += 1
              marker = CircleMarker((float(long), float(lat)), '#0036FF', 12)
              m.add_marker(marker)
        except requests.exceptions.HTTPError as e:
             print("Bad HTTP status code:", e)
        except requests.exceptions.RequestException as e:
             print("Network error:", e)
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

image = m.render(zoom=1)
image.save('mymap_zoom1.png')
image = m.render(zoom=2)
image.save('mymap_zoom2.png')
image = m.render(zoom=3)
image.save('mymap_zoom3.png')
image = m.render(zoom=4)
image.save('mymap_zoom4.png')
image = m.render(zoom=5)
image.save('mymap_zoom5.png')
image = m.render(zoom=6)
image.save('mymap_zoom6.png')

print ("Number total of request",nb_request)
print ("Number total of pin",nb_plot)

On same folder, you see mymap_zoom*.png .

To change the size of map, it’s here : ( by default 1600 x 800 )

m = StaticMap(1600, 800, url_template='http://a.tile.osm.org/{z}/{x}/{y}.png')

Time to build the pictures :

$ time python3 JoplinToStaticMap.py 
Number total of request 1927
Number total of pin 643

real	0m18.137s
user	0m7.936s
sys	0m0.876s

Joplin & Python ( gmplot Google Map) : How to create a Maps with REST API and data in Joplin ?

Step 0 : Install Joplin and activate the REST API ( https://joplin.cozic.net/api/ ) .

Step 1: Install gmplot with pip

$ pip install gmplot
Collecting gmplot
  Downloading https://files.pythonhosted.org/packages/e2/b1/e1429c31a40b3ef5840c16f78b506d03be9f27e517d3870a6fd0b356bd46/gmplot-1.2.0.tar.gz (115kB)
    100% |████████████████████████████████| 122kB 1.0MB/s 
Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (from gmplot) (2.21.0)
Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests->gmplot) (1.24.1)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests->gmplot) (2018.11.29)
Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests->gmplot) (2.8)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/site-packages (from requests->gmplot) (3.0.4)
Building wheels for collected packages: gmplot
  Building wheel for gmplot (setup.py) ... done
  Stored in directory: /Users/...../Library/Caches/pip/wheels/81/6a/76/4dd6a7cc310ba765894159ee84871e8cd55221d82ef14b81a1
Successfully built gmplot
Installing collected packages: gmplot
Successfully installed gmplot-1.2.0

The source code : (change your token)

#
# Version 1 
# for Python 3
# 
#   ARIAS Frederic
#   Sorry ... It's difficult for me the python :)
#

from time import gmtime, strftime
import time
import json
import requests
import os
import gmplot

strftime("%Y-%m-%d %H:%M:%S", gmtime())
start = time.time()

#Token
ip = "127.0.0.1"
port = "41184"
token = "Put your token here"

nb_request = 0
nb_plot = 0
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
url_notes = (
    "http://"+ip+":"+port+"/notes?"
    "token="+token
)
x=[] #longitudes
y=[] #latitudes
try:
    resp = requests.get(url_notes, headers=headers)
    nb_request += 1
    resp.raise_for_status()
    resp_dict = resp.json()
    #print(resp_dict)
    for my_note in resp_dict:
        #print(my_note.get('id'))
        url_notes2 = (
    "http://"+ip+":"+port+"/notes/"+my_note.get('id')+"?fields=longitude,latitude&"
    "token="+token
)
        try:
           resp2 = requests.get(url_notes2, headers=headers)
           nb_request += 1
           resp2.raise_for_status()
           resp_dict2 = resp2.json()
           #print(resp_dict2)
           long = resp_dict2.get('longitude')
           lat = resp_dict2.get('latitude')
           if (long != '0.00000000'):
              #print(long,lat)
              nb_plot += 1
              if (nb_plot == 1):
                 gmap1 = gmplot.GoogleMapPlotter(float(lat), float(long), 13 )
              x.append(float(long))
              y.append(float(lat))
        except requests.exceptions.HTTPError as e:
             print("Bad HTTP status code:", e)
        except requests.exceptions.RequestException as e:
             print("Network error:", e)
except requests.exceptions.HTTPError as e:
    print("Bad HTTP status code:", e)
except requests.exceptions.RequestException as e:
    print("Network error:", e)

gmap1.scatter( y, x, '#FF0000', size = 10, marker = False )
gmap1.draw("mymap.html")

print ("Number total of request",nb_request)
print ("Number total of pin",nb_plot)

After clic on mymap.html on same folder.

Note n°1 : JOPLIN is robust

 $ time python3 JoplinToGoogleMap.py 
Number total of request 1927
Number total of pin 643


real 0m8.401s
user 0m4.564s
sys 0m0.577s

In 9 seconds, 1927 requests !! JOPLIN it’s perfect.

Note n°2 : To increase the size of the PIN it’s here ( size = 10 ) :

gmap1.scatter( y, x, '#FF0000', size = 10, marker = False )