First pass for imports (with OFX imports)

src/model/Account.cpp

@@ -0,0 +1,34 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "Account.h"
+
+bool sortAccounts(Account ac1, Account ac2)
+{
+    if (ac1._default) return true;
+    if (ac2._default) return false;
+
+    if (!ac1.blocked && ac2.blocked) return true;
+    if (ac1.blocked && !ac2.blocked) return false;
+
+    if (!ac1._virtual && ac2._virtual) return true;
+    if (ac1._virtual && !ac2._virtual) return false;
+
+    return (ac1.name.Cmp(ac2.name) < 0);
+}

src/model/Account.h

@@ -36,4 +36,6 @@ public:
     bool _virtual;
 };
 
+bool sortAccounts(Account ac1, Account ac2);
+
 #endif

src/model/Database.h

@@ -42,6 +42,7 @@
 #define NON_FIX_OP     (1 << 1)
 #define CHECKED_OP     (1 << 2)
 #define NOT_CHECKED_OP (1 << 3)
+#define ALL_OP           (~0)
 
 // if (!_db.CheckSyntax(req))			
 // 	{

src/model/Operation.cpp

@@ -0,0 +1,48 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "Operation.h"
+
+bool sortOperations(Operation op1, Operation op2)
+{
+    if (!op1.fix_cost && op2.fix_cost) return false;
+    if (op1.fix_cost && !op2.fix_cost) return true;
+
+    if (op1.year < op2.year)
+	return true;
+    else if (op1.year == op2.year)
+    {
+	if (op1.month < op2.month)
+	    return true;
+	else if (op1.month == op2.month)
+	{
+	    if (op1.day < op2.day)
+		return true;
+	    else if (op1.day == op2.day)
+		return (op1.description.Cmp(op2.description) < 0);
+	}
+    }
+
+    return false;
+}
+
+bool reverseSortOperations(Operation op1, Operation op2)
+{
+    return !sortOperations(op1, op2);
+}

src/model/Operation.h

@@ -20,6 +20,9 @@
 #ifndef OPERATION_H
 #define OPERATION_H
 
+#include <wx/wx.h>
+#include <vector>
+
 class Operation {
 public:
     wxString id;
@@ -40,4 +43,7 @@ public:
     std::vector<wxString> childs;
 } ;
 
+bool sortOperations(Operation op1, Operation op2);
+bool reverseSortOperations(Operation op1, Operation op2);
+
 #endif

src/model/User.h

@@ -29,8 +29,10 @@
 #include "Account.h"
 #include "Operation.h"
 #include "Database.h"
+#include "import/ImportEngine.h"
 
 class Database;
+class ImportPattern;
 
 class User
 {
@@ -47,6 +49,7 @@ public:
     std::vector<Category> _categories;
     std::vector<wxFont> _categoriesFonts;
     std::map<wxString, wxString> _preferences;
+    std::map<wxString, ImportPattern> _importPatterns;
 
     Category GetCategory(const wxString& catId);
     wxString GetCategoryName(const wxString& catId);

src/model/import/ImportEngine.cpp

@@ -0,0 +1,269 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "ImportEngine.h"
+
+ImportEngine::ImportEngine()
+{
+}
+
+ImportEngine::~ImportEngine()
+{
+}
+
+bool ImportEngine::HandleFile(const wxString& path, User* user, Database* db, KissCount* kiss)
+{
+    _path = path;
+    _user = user;
+    _db = db;
+    _kiss = kiss;
+
+    _accounts.clear();
+    _unresolvedAccounts.clear();
+    _operations.clear();
+    _descriptions.clear();
+
+    return path.EndsWith(_shortExt) || path.EndsWith(_shortExt.Upper());
+}
+
+wxString ImportEngine::GetFileExt()
+{
+    return wxGetTranslation(_longExt);
+}
+
+std::vector<wxString> ExplodeString(wxString& s)
+{
+    wxString tmp = s, cur = wxT("");
+    std::vector<wxString> res;
+
+    while (tmp.Len())
+    {
+	if (tmp.StartsWith(wxT(" ")) || 
+	    tmp.StartsWith(wxT("\t")) || 
+	    tmp.StartsWith(wxT("\n")) || 
+	    tmp.StartsWith(wxT("\r")))
+	{
+	    if (cur.Len())
+	    {
+		res.push_back(cur);
+		cur = wxT("");
+	    }
+	    tmp = tmp.Mid(1);
+	    continue;
+	}
+	cur += tmp.SubString(0, 0);
+	tmp = tmp.Mid(1);
+    }
+
+    if (cur.Len())
+	res.push_back(cur);
+
+    return res;
+}
+
+/*
+  Remove :
+  - head spaces
+  - tail spaces
+  - every word starting by a number
+ */
+wxString ImportEngine::RemoveUnused(wxString& s)
+{
+    wxString tmp = s, tmp2;
+    wxString res;
+
+    tmp = tmp.Trim(true);
+    tmp = tmp.Trim(false);
+
+    while (tmp.Len())
+    {
+	tmp2 = tmp.SubString(0, 0);
+	if (tmp2.IsNumber())
+	{
+	    do
+	    {
+		tmp = tmp.Mid(1);
+		tmp2 = tmp.SubString(0, 0);
+	    } while (tmp.Len() && tmp2 != wxT(" ") &&
+		     tmp2 != wxT("\t") &&
+		     tmp2 != wxT("\r") && 
+		     tmp2 != wxT("\n"));
+	}
+	res += tmp2;
+	tmp = tmp.Mid(1);
+    }
+
+    return res;
+}
+
+/*
+  Find a pattern between the two strings:
+  %mX : lower string of orig[X]
+  %MX : upper string of orig[X]
+  %fX : First case in upper case of orig[X]
+  %wX : word orig[X]
+  other : constants
+ */
+wxString ImportEngine::FindPattern(wxString& orig, wxString& dest)
+{
+    wxString pattern, cur_pat;
+    int i, a;
+    std::vector<wxString> tok1 = ExplodeString(orig);
+    std::vector<wxString> tok2 = ExplodeString(dest);
+    int size1 = tok1.size(), size2 = tok2.size();
+    
+    for(i=0; i<size2; i++)
+    {
+	cur_pat = wxT("");
+
+	for (a=0; a<size1; a++)
+	{
+	    if (tok2[i] == tok1[a])
+	    {
+		cur_pat = wxT("%w");
+		break;
+	    }
+	    else if (tok2[i] == tok1[a].Lower())
+	    {
+		cur_pat = wxT("%m");
+		break;
+	    }
+	    else if (tok2[i] == tok1[a].Upper())
+	    {
+		cur_pat = wxT("%m");
+		break;
+	    }
+	    else if (tok2[i].Lower() == tok1[a].Lower() && tok2[i][0] == tok1[a][0])
+	    {
+		cur_pat = wxT("%f");
+		break;
+	    }
+	}
+
+	if (cur_pat.Len())
+	    pattern += cur_pat + wxString::Format(wxT("%d"), a);
+	else
+	    pattern += tok2[i];
+
+	pattern += wxT(" ");
+    }
+
+    pattern = pattern.Trim();
+
+    return pattern;
+}
+
+void ImportEngine::ApplyPattern(ImportPattern& pattern, Operation& op)
+{
+    std::vector<wxString> tok1 = ExplodeString(pattern.filter);
+    std::vector<wxString> tok2 = ExplodeString(op.description);
+    int size1 = tok1.size(), i;
+    long pos;
+
+    op.description = wxT("");
+
+    for(i=0; i<size1; i++)
+    {
+	pos = -1;
+	if (tok1[i].StartsWith(wxT("%")))
+	{
+	    tok1[i].Mid(3).ToLong(&pos);
+	}
+
+	if (pos != -1)
+	{
+	    if (tok1[i].SubString(1, 1) == wxT("w"))
+		op.description += tok2[pos];
+	    else if (tok1[i].SubString(1, 1) == wxT("m"))
+		op.description += tok2[pos].Lower();
+	    else if (tok1[i].SubString(1, 1) == wxT("M"))
+		op.description += tok2[pos].Upper();
+	    else if (tok1[i].SubString(1, 1) == wxT("f"))
+		op.description += tok2[pos].SubString(0, 0).Upper() + tok2[pos].Mid(1).Lower();
+	}
+	else
+	    op.description += tok1[i];
+	op.description += wxT(" ");
+    }
+
+    op.description.Trim();
+    op.account = pattern.account;
+    op.category = pattern.category;
+}
+
+int ImportEngine::UpdatePattern(int pos)
+{
+    wxString key1, key2;
+    ImportPattern pattern;
+    Operation op;
+    int i, nbOpUpdated = 0;
+
+    if (!_user) return 0;
+
+    nbOpUpdated = 1;
+
+    op = _operations[pos];
+
+    key1 = RemoveUnused(_descriptions[op.id]);
+
+    pattern.filter = FindPattern(_descriptions[op.id], op.description);
+    pattern.account = op.account;
+    pattern.category = op.category;
+
+    _user->_importPatterns[key1] = pattern;
+
+    for(i=pos+1; i<(int)_operations.size(); i++)
+    {
+	key2 = RemoveUnused(_descriptions[_operations[i].id]);
+
+	if (key1 == key2)
+	{
+	    ApplyPattern(_user->_importPatterns[key2], _operations[i]);
+	    nbOpUpdated++;
+	}
+    }
+
+    return nbOpUpdated;
+}
+
+void ImportEngine::MatchPattern(wxString& originalKey, Operation& op)
+{
+    wxString key1;
+    ImportPattern pattern;
+
+    if (!_user) return;
+
+    key1 = RemoveUnused(originalKey);
+
+    if (!_user->_importPatterns.count(key1))
+    {
+	pattern.filter = FindPattern(originalKey, op.description);
+	pattern.account = op.account;
+	pattern.category = op.category;
+
+	_user->_importPatterns[key1] = pattern;
+
+	// std::cout << "New pattern " << key1.mb_str() << "\t" << pattern.filter.mb_str() << std::endl;
+    }
+    else
+    {
+	op.description = _descriptions[op.id];
+	ApplyPattern(_user->_importPatterns[key1], op);
+    }
+}

src/model/import/ImportEngine.h

@@ -0,0 +1,75 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef IMPORTENGINE_H
+#define IMPORTENGINE_H
+
+#include <model/model.h>
+#include <controller/KissCount.h>
+
+class KissCount;
+
+class ImportPattern {
+public:
+    wxString filter;
+    wxString account;
+    wxString category;
+} ;
+
+class ImportEngine {
+public:
+    ImportEngine();
+    ~ImportEngine();
+
+    // Get supported file extension. Example :
+    // "OFX files (*.ofx)|*.ofx"
+    virtual wxString GetFileExt();
+
+    // Handle the file
+    virtual bool HandleFile(const wxString& path, User* user, Database* db, KissCount* kiss);
+
+    // Parse the file and return accounts that doesn't match
+    virtual std::vector<wxString> ParseFile()=0;
+    
+    // Final Step
+    virtual std::vector<Operation>* GetOperations(std::map<wxString, wxString>& accounts)=0;
+
+    void MatchPattern(wxString& key, Operation& op);
+    int UpdatePattern(int pos);
+
+protected:
+    Database* _db;
+    User* _user;
+    wxString _path;
+    KissCount* _kiss;
+
+    wxString _shortExt;
+    wxString _longExt;
+
+    std::map<wxString, wxString> _accounts;
+    std::vector<wxString> _unresolvedAccounts;
+    std::vector<Operation> _operations;
+    std::map<wxString, wxString> _descriptions;
+
+    wxString RemoveUnused(wxString& s);
+    void ApplyPattern(ImportPattern& pattern, Operation& op);
+    wxString FindPattern(wxString& s1, wxString& s2);
+};
+
+#endif

src/model/import/OFXImportEngine.cpp

@@ -0,0 +1,153 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "OFXImportEngine.h"
+
+static OFXImportEngine ofxImportEngine;
+
+int OFXImportEngine::account_cb(const struct OfxAccountData data, void * account_data)
+{
+    int i;
+    OFXImportEngine* _this = (OFXImportEngine*) account_data;
+    wxString account_number = wxString(data.account_number, wxConvUTF8);
+
+    _this->_curAccount = wxT("");
+
+    for (i=0; i<(int)_this->_user->_accounts.size(); i++)
+    {
+	if (_this->_user->_accounts[i].number == account_number)
+	{
+	    _this->_accounts[account_number] = _this->_user->_accounts[i].id;
+	    _this->_curAccount = _this->_user->_accounts[i].id;
+	    // std::cout << "Account " << data.account_number << " is " << i << std::endl;
+	    //_this->_unresolvedAccounts.push_back(account_number);
+	    break;
+	}
+    }
+
+    if (!_this->_curAccount.Len())
+    {
+	_this->_accounts[account_number] = wxT("unknown-") + account_number;
+	_this->_unresolvedAccounts.push_back(account_number);
+    }
+
+    return 0;
+}
+
+int OFXImportEngine::transaction_cb(const struct OfxTransactionData data, void * transaction_data)
+{
+    static int id=0;
+    OFXImportEngine* _this = (OFXImportEngine*) transaction_data;
+    Operation op;
+    struct tm t;
+
+    if (!data.amount_valid ||
+	(!data.date_posted_valid && !data.date_initiated_valid)) return 1;
+
+    op.id = wxString::Format(wxT("%d"), ++id);
+    op.parent = wxT("");
+    op.category = wxT("0");
+    op.fix_cost = false;
+    op.account = _this->_curAccount;
+    op.checked = false;
+    op.transfert = wxT("");
+    op.formula = wxT("");
+    op.meta = false;
+    op._virtual = false;
+
+    op.amount = data.amount;
+
+    if (data.date_initiated_valid)
+	gmtime_r(&data.date_initiated, &t);
+    else
+	gmtime_r(&data.date_posted, &t);
+
+    op.day = t.tm_mday;
+    op.month = t.tm_mon;
+    op.year = t.tm_year+1900;
+
+    if (data.name_valid)
+	op.description = wxString(data.name, wxConvUTF8);
+
+    if (data.memo_valid)
+    {
+	if (op.description.Len())
+	    op.description += wxT(" ");
+	op.description += wxString(data.memo, wxConvUTF8);
+    }
+
+    _this->_operations.push_back(op);
+    _this->_descriptions[op.id] = op.description;
+
+    _this->MatchPattern(op.description, op);
+
+    return 0;
+}
+
+int OFXImportEngine::account_balance_cb(const struct OfxStatementData data, void * statement_data)
+{
+    OFXImportEngine* _this = (OFXImportEngine*) statement_data;
+    
+    return 0;
+}
+
+OFXImportEngine::OFXImportEngine()
+{
+    KissCount::RegisterImportEngine(this);
+
+    _ctx = libofx_get_new_context();
+
+    if (!_ctx)
+	throw std::string("Unable to initialize libofx");
+
+    ofx_set_account_cb(_ctx, account_cb, this);
+    ofx_set_transaction_cb(_ctx, transaction_cb, this);
+    ofx_set_statement_cb(_ctx, account_balance_cb, this);
+
+    _shortExt = wxT("ofx");
+    _longExt = _("OFX files (*.ofx)|*.ofx");
+}
+
+OFXImportEngine::~OFXImportEngine()
+{
+    if (_ctx)
+	libofx_free_context(_ctx);
+}
+
+bool OFXImportEngine::HandleFile(const wxString& path, User* user, Database* db, KissCount* kiss)
+{
+    if (!ImportEngine::HandleFile(path, user, db, kiss)) return false;
+
+    return !libofx_proc_file(_ctx, path.mb_str(), AUTODETECT);
+}
+
+std::vector<wxString> OFXImportEngine::ParseFile()
+{
+    return _unresolvedAccounts;
+}
+
+std::vector<Operation>* OFXImportEngine::GetOperations(std::map<wxString, wxString>& accounts)
+{
+    if (_kiss->GetOperationOrder() == wxT("ASC"))
+	std::sort(_operations.begin(), _operations.end(), sortOperations);
+    else
+	std::sort(_operations.begin(), _operations.end(), reverseSortOperations);
+ 
+    return &_operations;
+}

src/model/import/OFXImportEngine.h

@@ -0,0 +1,44 @@
+/*
+  Copyright 2010-2011 Grégory Soutadé
+
+  This file is part of KissCount.
+
+  KissCount is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  KissCount is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with KissCount.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef OFXIMPORTENGINE_H
+#define OFXIMPORTENGINE_H
+
+#include "ImportEngine.h"
+#include <libofx/libofx.h>
+
+class OFXImportEngine : public ImportEngine {
+public:
+    OFXImportEngine();
+    ~OFXImportEngine();
+
+    virtual bool HandleFile(const wxString& path, User* user, Database* db, KissCount* kiss);
+    virtual std::vector<wxString> ParseFile();
+    virtual std::vector<Operation>* GetOperations(std::map<wxString, wxString>& accounts);
+
+private:
+    LibofxContextPtr _ctx;
+    wxString _curAccount;
+
+    static int account_cb(const struct OfxAccountData data, void * account_data);
+    static int transaction_cb(const struct OfxTransactionData data, void * transaction_data);
+    static int account_balance_cb(const struct OfxStatementData data, void * statement_data);
+};
+
+#endif