Denormalize database and handle multiple locations with same name

.gitignore

@@ -0,0 +1 @@
+*.pyc

00-init-db.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS `locations` (
+  `id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+  `name` VARCHAR(100) NOT NULL,
+  `country` VARCHAR(100) NOT NULL,
+  `xml` VARCHAR(200) NOT NULL,
+  UNIQUE (name, country) ON CONFLICT REPLACE
+);
+
+CREATE INDEX locations_name ON locations ('name');
+CREATE INDEX locations_country ON locations ('country');
+

01-parse-data.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 
-#This file is used to parse location data with yr.no-URLs.
-#We will end up with countries.txt, verda2.txt. Import in db with 02-import-data.py
+# This file is used to parse location data with yr.no-URLs.
+# Output is pushed into all_locations.txt. 
+# Import file with 02-import-data.py
 
-echo "parse world, forcast every 6 hours (not by choice)"
-cut -f1,11 --output-delimiter=, verda.txt |sort |uniq > countries.txt
-cut -f1,4,11,18 --output-delimiter=, verda.txt > verda2.txt
+echo "Parsing norwegian locations - hourly forecast available"
+tail -n+2 noreg.txt |cut -f 2,14 --output-delimiter=';' |awk 'BEGIN {FS =";"} {print $1 ",Norway,"$2 }' |sed 's/\/forecast.xml/\/forecast_hour_by_hour.xml/g' > all_locations.txt
 
-echo "parse norway, hourly forcast"
-tail -n+2 noreg.txt |cut -f 2,14 --output-delimiter=, |awk 'BEGIN { FS = "," } {print "NO,",$1,",Norway,",$2 }' |sed 's/\/forecast.xml/\/forecast_hour_by_hour.xml/g' >> verda2.txt
-echo "NO,Norway" >> countries.txt
+echo "Parsing international locations - forecast every 6 hours (not by choice)"
+cut -f4,11,18 --output-delimiter=, verda.txt >> all_locations.txt

02-import-data.py

@@ -2,82 +2,25 @@
 # -*- coding: UTF-8; -*-
 
 '''
-Put verda2.txt in mysql db
+Import all_locations.txt into locations.db
 '''
 
-import string, MySQLdb, sys
+import string, sys
+import sqlite3 as lite
 
-def get_db_cursor ():
-  conn=MySQLdb.connect(host = "localhost",
-                           user = "pyyrlib",
-                           passwd = "ifoo3aeshahN",
-                           db = "pyyrlib")
-  return conn, conn.cursor ()
+conn = lite.connect('locations.db')
+conn.text_factory = str
+cursor = conn.cursor()
 
+cursor.execute("DELETE FROM locations")
 
-def clear_db (cursor, table):
-  query = "delete from " + table
-  cursor.execute(query)
+query = "INSERT INTO locations (name, country, xml) VALUES (?, ?, ?);"
+fd = open( "all_locations.txt" )
+content = fd.readline()
+while (content != "" ):
+  fields = string.split(content, ',')
+  cursor.execute(query, (fields[0], fields[1].strip(), fields[2].strip()))
+  content = fd.readline()  
+conn.commit()
+conn.close()
 
-
-def insert_row_countries (cursor, table, fields):
-  query = "INSERT INTO " + table + " (countrycode, countryname) VALUES ( "
-
-  for i in range(0, 2):
-    if 0 != i:
-      query += ", "
-    query += "'" + all_lower(fields[i]) + "'"
-
-  query += " ) ON DUPLICATE KEY UPDATE countryname = '" + fields[1] + "' ;"
-
-  print query
-  return cursor.execute(query)
-
-
-def insert_row_verda (cursor, conn, table, fields):
-  query = "INSERT INTO " + table + " (countryid, placename, xml) VALUES ( "
-
-  for i in [0, 1, 3]:
-    if 0 != i:
-      query += ", "
-    if 0 == i:
-      query += " (select countryid from countries where countrycode = '" + all_lower(fields[0]) + "' ) "
-    elif 2 == i:
-      continue
-    else:
-      query += "'" + conn.escape_string(all_lower(fields[i].replace(' ', ''))) + "'"
-
-  query += " ) ;"
-
-  print query
-  return cursor.execute(query)
-
-
-def process_file_countries (cursor):
-  fd = open( "countries.txt" )
-  content = fd.readline()
-  while (content != "" ):
-    fields = string.split(content, ',')
-    insert_row_countries(cursor, 'countries', fields)
-    content = fd.readline()
-
-
-def process_file_verda (cursor, conn):
-  fd = open( "verda2.txt" )
-  content = fd.readline() #header
-  content = fd.readline()
-  while (content != "" ):
-    fields = string.split(content, ',')
-    insert_row_verda(cursor, conn, 'verda', fields)
-    content = fd.readline()
-
-
-def all_lower (str):
-  return str.strip().lower().replace('Æ', 'æ').replace('Ø', 'ø').replace('Å', 'å')
-
-
-conn, c = get_db_cursor ()
-clear_db (c, 'countries')
-process_file_countries (c)
-clear_db (c, 'verda')
-process_file_verda(c, conn)

pyofc.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8; -*-
+
+__version__ = '0.1'
+__url__ = 'https://github.com/ways/pyofflinefilecache'
+__license__ = 'GPL License'
+
+import os, time, datetime, codecs
+
+
+class OfflineFileCache:
+  ''' A class storing data as files for later retreval based on timestamp.
+      First line of cache file contains timestamp. Rest is text data. '''
+
+  def __init__(self, cachedir, cachetime, fetchfunction, fetcharg = False, verbose = False):
+    self.fetchfunction = fetchfunction
+    self.fetcharg = fetcharg
+    self.cachetime = cachetime
+    self.cachedir = cachedir
+    self.verbose = verbose
+
+    if not os.path.exists(self.cachedir):
+      try:
+        os.mkdir(self.cachedir, 0700)
+      except FileError as e:
+        print "Error writing to dir ", self.cachedir, e
+
+
+  def set(self, id, data):
+    id = self.escape_string(id)
+    #print data.read()
+    try:
+      #with codecs.open(self.cachedir + id, 'w', encoding='utf-8') as f:
+      with codecs.open(self.cachedir + id, 'w') as f:
+        f.write(str(time.time()) + "\n" + data )
+    except IOError as e:
+      print "OfflineFileCache: Error opening file " + self.cachedir + id
+      #return False
+
+  def status(self, id):
+    id = self.escape_string(id)
+    try:
+      with codecs.open(self.cachedir + id, 'r') as f:
+        firstline = f.readline()
+        filetime = float(firstline)
+    except IOError as e:
+      if self.verbose:
+        print "Error reading file " + self.cachedir + id + str(e)
+      return False
+    except ValueError as e:
+      if self.verbose:
+        print "Error converting time from " + str(firstline)
+        return False
+
+    if self.verbose:
+      print "Time from file: " + str(datetime.datetime.fromtimestamp(filetime))
+
+    if filetime > (time.time() - self.cachetime):
+      if self.verbose:
+        print "File is fresh, remaining " + str((filetime + self.cachetime) - time.time())
+      return True
+    else:
+      if self.verbose:
+        print "File is sour, over time: " + str(time.time() - filetime)
+      return False
+
+
+  def get(self, id):
+    id = self.unescape_string(id)
+    if self.status(id):
+      #with codecs.open(self.cachedir + id, 'r', encoding='utf-8') as f:
+      with codecs.open(self.cachedir + id, 'r') as f:
+        filetime = f.readline()
+        if self.verbose:
+          print "Filetime ", filetime
+          print "Returning cached data for", id
+        return f.read(), True
+    else:
+      if self.verbose:
+        print "Resetting contents of file"
+      data = self.fetchfunction(self.fetcharg)
+
+      self.set(id, data)
+
+      if self.verbose:
+        print "Returning fresh data for", id
+        #print data
+      return data, False
+
+
+  def escape_string(self, str):
+ #   if len(str) > 20:
+#      str = str[:20]
+#    str = str.strip()\
+#    .replace('..','')\
+#    .replace('~','£')\
+#    .replace('/','_')
+
+    return str
+
+  def unescape_string(self, str):
+#    str = str\
+#    .replace('£','~')\
+#    .replace('_','/')
+
+    return str
+
+if __name__ == "__main__":
+  #Example data:
+  cachedir="/tmp/pyyrlib-cache/"
+  cachetime=1200
+
+  def fetchdata(id = "0000"):
+    return "It's sunny at " + id
+
+  #Example usage:
+  ofc = OfflineFileCache (cachedir, cachetime, fetchdata, "0459", True)
+  data, fromcache = ofc.get('0459')
+  print "data: " + str(data)