net_blast_ui_data_source.cpp

Go to the documentation of this file.

/*  $Id: net_blast_ui_data_source.cpp 25238 2012-02-14 16:11:43Z ucko $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Andrey Yazhuk, Anatoliy Kuznetsov
 *
 */

#include <ncbi_pch.hpp>

#include <corelib/ncbi_system.hpp>

#include <gui/packages/pkg_alignment/net_blast_ui_data_source.hpp>

#include <gui/packages/pkg_alignment/net_blast_ui_load_manager.hpp>
#include <gui/packages/pkg_alignment/blast_search_task.hpp>

#include <gui/core/app_explorer_service.hpp>
#include <gui/core/data_mining_service.hpp>
#include <gui/core/data_mining_view.hpp>
#include <gui/core/app_tasks.hpp>
#include <gui/core/download_job.hpp>

#include <gui/utils/app_job_impl.hpp>

#include <gui/framework/workbench.hpp>
#include <gui/framework/app_task_service.hpp>
#include <gui/framework/view_manager_service.hpp>

#include <gui/widgets/wx/ui_command.hpp>

#include <gui/objutils/registry.hpp>

#include <gui/utils/extension_impl.hpp>
#include <gui/widgets/wx/sys_path.hpp>

#include <objects/general/Object_id.hpp>

#include <objects/blast/blastclient.hpp>
#include <objects/blast/Blast4_database.hpp>
#include <objects/blast/Blast4_database_info.hpp>
#include <objects/taxon1/taxon1.hpp>

#include <wx/menu.h>
#include <wx/msgdlg.h>
#include <wx/stopwatch.h>
#include <wx/filename.h>
#include <wx/datetime.h>

//#include <gui/packages/pkg_alignment/blast_winmask_dialog.hpp>

BEGIN_NCBI_SCOPE
USING_SCOPE(objects);

///////////////////////////////////////////////////////////////////////////////
///

class CNetBlastLoadDBListJob : public CAppJob
{
public:
    CNetBlastLoadDBListJob(CNetBLASTUIDataSource* blast_ds);
    ~CNetBlastLoadDBListJob();

    virtual EJobState   Run();
private:
    CRef<CNetBLASTUIDataSource>                  m_BlastDS;
    CRef<objects::CBlast4_get_databases_reply>   m_Databases;
};

CNetBlastLoadDBListJob::CNetBlastLoadDBListJob(CNetBLASTUIDataSource* blast_ds)
: m_BlastDS(blast_ds)
{
}


CNetBlastLoadDBListJob::~CNetBlastLoadDBListJob()
{
}

IAppJob::EJobState   CNetBlastLoadDBListJob::Run()
{
    try {
        CBlast4Client client;
        client.SetRetryLimit(2); // TODO: make it configurable
        m_Databases = client.AskGet_databases();

        if (m_Databases) {
            /// serialize the last successful result - this will be used
            /// the next time through if the call to retrieve the databases fails
            wxString path = CSysPath::ResolvePath( wxT("<home>/blast-dbs.asn") );
            CNcbiOfstream ostr(path.fn_str(),
                               ios::out | ios::binary);
            ostr << MSerial_AsnBinary << *m_Databases;
        }
        
        // attach database list to the data source
        m_BlastDS->x_SetDataBases(m_Databases);
    }
    catch (CException&) {
        m_Databases.Reset();
        m_BlastDS->x_SetDBLoadJobId(0);
        return eFailed;
    }
    return eCompleted;
}


///////////////////////////////////////////////////////////////////////////////
/// CNetBlastDSEvtHandler - wxEvtHandler-derived adapter for Net BLAST data source.
/// This class translates menu commands into Net BLAST data source method calls.

class CNetBlastDSEvtHandler : public wxEvtHandler
{
    DECLARE_EVENT_TABLE();
public:
    CNetBlastDSEvtHandler(CNetBLASTUIDataSource& ds)
        :   m_NetBlastDS(&ds)
    {};

    CNetBlastDSEvtHandler(CNetBLASTUIDataSource& ds, CNetBLASTUIDataSource::TJobDescrVec& jobs)
        :   m_NetBlastDS(&ds), m_Jobs(jobs)
    {};

    ~CNetBlastDSEvtHandler()
    {
    }

    void OnRetrieve(wxCommandEvent& event)
    {
        m_NetBlastDS->StartRetrievingTask(m_Jobs);
    }
    void OnMonitor(wxCommandEvent& event)
    {
        m_NetBlastDS->StartMonitoringTask(m_Jobs);
    }
    void OnDelete(wxCommandEvent& event)
    {
        m_NetBlastDS->DeleteJobDescriptors(m_Jobs);
    }
    void OnExplore(wxCommandEvent& event)
    {
        m_NetBlastDS->OnExplore();
    }
    void OnLoadRIDs(wxCommandEvent& event)
    {
        m_NetBlastDS->OnLoadRIDs();
    }

public:
    CRef<CNetBLASTUIDataSource> m_NetBlastDS;
    CNetBLASTUIDataSource::TJobDescrVec m_Jobs;
};

BEGIN_EVENT_TABLE(CNetBlastDSEvtHandler, wxEvtHandler)
    EVT_MENU(eCmdRetriveBlastJobs, CNetBlastDSEvtHandler::OnRetrieve)
    EVT_MENU(eCmdMonitorBlastJobs, CNetBlastDSEvtHandler::OnMonitor)
    EVT_MENU(eCmdDeleteBlastJobs, CNetBlastDSEvtHandler::OnDelete)
    EVT_MENU(eCmdExploreNetBLAST, CNetBlastDSEvtHandler::OnExplore)
    EVT_MENU(eCmdLoadRIDs, CNetBlastDSEvtHandler::OnLoadRIDs)
END_EVENT_TABLE();

///////////////////////////////////////////////////////////////////////////////
/// CNetBLASTUIDataSource

static const char* kNetBLAST_DS_Icon = "icon::gb_data_source"; //TODO
//static const char* kNetBlastCmdContributor = "net_blast_cmd_contributor";
static const char* kCmdExtPoint = "scoped_objects::cmd_contributor";
static const char* kAppExpCmdExtPoint = "project_tree_view::context_menu::item_cmd_contributor";


CNetBLASTUIDataSource::CNetBLASTUIDataSource(CNetBLASTUIDataSourceType& type)
:   m_Descr("NCBI Net BLAST", kNetBLAST_DS_Icon),
    m_Type(&type),
    m_SrvLocator(NULL),
    m_Open(false)
{
    m_DBLoadJobId = 0;
}


CNetBLASTUIDataSource::~CNetBLASTUIDataSource()
{
    _ASSERT(! m_Open);
}


string CNetBLASTUIDataSource::GetExtensionIdentifier() const
{
    return "net_blast_data_source";
}


string CNetBLASTUIDataSource::GetExtensionLabel() const
{
    return "Net BLAST Data Source";
}


void CNetBLASTUIDataSource::SetServiceLocator(IServiceLocator* locator)
{
    m_SrvLocator = locator;
}

//static const char* kCNetBLASTUIRootKey = "NetBlastUI";
//static const char* kLocalMaskPath = "LocalMaskPath";
//static const char* kRemoteMaskURL = "RemoteMaskURL";

void CNetBLASTUIDataSource::SetRegistryPath( const string& path )
{
    m_RegPath = path;
}

void CNetBLASTUIDataSource::SaveSettings() const
{
    _ASSERT(! m_RegPath.empty() );

    if( !m_RegPath.empty() ){
        CGuiRegistry& gui_reg = CGuiRegistry::GetInstance();
        CGuiRegistry::TReadWriteView view = gui_reg.GetReadWriteView(m_RegPath);
    }
}


void CNetBLASTUIDataSource::LoadSettings()
{
    _ASSERT(! m_RegPath.empty());

    if( ! m_RegPath.empty())   {
        CGuiRegistry& gui_reg = CGuiRegistry::GetInstance();
        CGuiRegistry::TReadView view = gui_reg.GetReadView(m_RegPath);
    }
}

IUIDataSourceType& CNetBLASTUIDataSource::GetType() const
{
    return *m_Type;
}


const IUIObject& CNetBLASTUIDataSource::GetDescr()
{
    return m_Descr;
}


bool CNetBLASTUIDataSource::IsOpen()
{
    return m_Open;
}


bool CNetBLASTUIDataSource::Open()
{
    if ( !m_Open )   {
        ERR_POST(Info <<"Opening Net BLAST Data Source...");

        CStopWatch sw;
        sw.Start();

        /// register itself as menu contributor
        CIRef<IExtensionRegistry> reg = CExtensionRegistry::GetInstance();
        reg->AddExtension(kCmdExtPoint, *this);
        reg->AddExtension(kAppExpCmdExtPoint, *this);

        x_InitDatabases();

        x_LoadJobDescriptors();

        x_AutoStartMonitoring();

        m_Open = true;

        string t = NStr::DoubleToString(sw.Elapsed(), 3);
        ERR_POST(Info << "Registered Net BLAST Data Source - " << t << " sec");
        return true;
    } else {
        return false;
    }
}


bool CNetBLASTUIDataSource::Close()
{
    if(m_Open)  {
        x_SaveJobDescriptors();
        m_JobDescrs.clear();

        x_ClearDatabases();

        /// remove itself from menu contribution points
        CIRef<IExtensionRegistry> reg = CExtensionRegistry::GetInstance();
        reg->RemoveExtension(kCmdExtPoint, *this);
        reg->RemoveExtension(kAppExpCmdExtPoint, *this);

        m_Open = false;
        return true;
    } else return false;
}


void CNetBLASTUIDataSource::EditProperties()
{
    //TODO
}


IUIToolManager* CNetBLASTUIDataSource::GetLoadManager()
{
    // TODO may need to link the manager to this particular datasource
    return new CNetBLASTUILoadManager(*this); //TODO ?
}

IObjectCmdContributor::TContribution
    CNetBLASTUIDataSource::GetMenu(TConstScopedObjects& objects)
{
    wxMenu* menu = NULL;
    wxEvtHandler* handler = NULL;

    if( ! objects.empty()) {
        // examine data
        int retrieve = 0;
        int monitor = 0;
        CNetBLASTUIDataSource::TJobDescrVec jobs;

        NON_CONST_ITERATE(TConstScopedObjects, it, objects) {
            const CObject* obj = it->object.GetPointer();
            const CNetBlastJobDescriptor* const_descr =
                dynamic_cast<const CNetBlastJobDescriptor*>(obj);
            CNetBlastJobDescriptor* descr = const_cast<CNetBlastJobDescriptor*>(const_descr);

            if(descr)   {
                jobs.push_back(CRef<CNetBlastJobDescriptor>(descr));

                CNetBlastJobDescriptor::EState state = descr->GetState();

                switch(state)   {
                case CNetBlastJobDescriptor::eCompleted:
                case CNetBlastJobDescriptor::eRetrieved:
                    retrieve++;
                    break;
                case CNetBlastJobDescriptor::eSubmitted:
                    monitor++;
                    break;
                default:
                    break;
                }
            }
        }

        CUICommandRegistry& cmd_reg = CUICommandRegistry::GetInstance();

        menu = new wxMenu;
        menu->Append(wxID_SEPARATOR, wxT("Top Actions"));
        if(retrieve > 0)    {
            cmd_reg.AppendMenuItem(*menu, eCmdRetriveBlastJobs);
        }
        if(monitor > 0)    {
            cmd_reg.AppendMenuItem(*menu, eCmdMonitorBlastJobs);
        }
        if( ! jobs.empty()) {
            cmd_reg.AppendMenuItem(*menu, eCmdDeleteBlastJobs);
            handler = new CNetBlastDSEvtHandler(*this, jobs);
        }
    }

    return IObjectCmdContributor::TContribution(menu, handler);
}


IExplorerItemCmdContributor::TContribution
    CNetBLASTUIDataSource::GetMenu(TItemRefVector& items,
                                   CAppExplorerService& app_service)
{
    IExplorerItemCmdContributor::TContribution contrib;
    if(items.size() != 1)   {
        return contrib;
    }

    CUICommandRegistry& cmd_reg = CUICommandRegistry::GetInstance();

    CExplorerItem* item = items[0].GetPointer();
    int type = app_service.GetItemType(*item);

    if(type == CAppExplorerService::eDataSource)    {
        CAppExplorerService::TDataSourceTreeItem* ds_item =
            dynamic_cast<CAppExplorerService::TDataSourceTreeItem*>(item);

        if(ds_item) {
            CIRef<IUIDataSource> ds = ds_item->GetData();
            CNetBLASTUIDataSource* blast_ds = dynamic_cast<CNetBLASTUIDataSource*>(ds.GetPointer());

            if(blast_ds)  {
                wxMenu* menu = new wxMenu;
                menu->Append(wxID_SEPARATOR, wxT("Top Actions"));
                cmd_reg.AppendMenuItem(*menu, eCmdExploreNetBLAST);
                cmd_reg.AppendMenuItem(*menu, eCmdLoadRIDs);

                contrib.first = menu;
                contrib.second = new CNetBlastDSEvtHandler(*this);
            }
        }
    }
    return contrib;
}


const CNetBLASTUIDataSource::TDbMap& CNetBLASTUIDataSource::GetDbMap(bool nuc) const
{
    {
    CFastMutexGuard guard(m_DBLoadMutex);
    return nuc ? m_NucDbMap : m_ProtDbMap;
    }
}


string CNetBLASTUIDataSource::GetDefaultDB(bool nuc_db) const
{
    {
    CFastMutexGuard guard(m_DBLoadMutex);
    return nuc_db ? m_Nuc_DefDB : m_Prot_DefDB;
    }
}

void CNetBLASTUIDataSource::WaitReady()
{
    x_WaitForDBLoadJob();
}

void CNetBLASTUIDataSource::GetDefaultMRU_DBs(bool nuc_db, list<string>& mru_dbs)
{
    const vector<string>& dbs = nuc_db ? m_Nuc_DefMRU_DBs : m_Prot_DefMRU_DBs;
    ITERATE(vector<string>, it, dbs)    {
        mru_dbs.push_back(*it);
    }
}


CRef<CNetBlastJobDescriptor>
    CNetBLASTUIDataSource::CreateJobDescriptor(const CBLASTParams& params)
{
    CFastMutexGuard guard(m_JobDescrMutex);

    CRef<CNetBlastJobDescriptor> descr(new CNetBlastJobDescriptor());
    descr->Init(params, *this);
    m_JobDescrs.push_back(descr);
    return descr;
}


CRef<CNetBlastJobDescriptor>
    CNetBLASTUIDataSource::CreateJobDescriptor(const string& rid)
{
    CFastMutexGuard guard(m_JobDescrMutex);

    CRef<CNetBlastJobDescriptor> descr(new CNetBlastJobDescriptor());
    descr->Init(rid, *this);
    m_JobDescrs.push_back(descr);
    return descr;
}


void CNetBLASTUIDataSource::GetJobDescriptors(vector< CRef<CNetBlastJobDescriptor> >& descriptors)
{
    CFastMutexGuard guard(m_JobDescrMutex);

    for(  size_t i = 0;  i < m_JobDescrs.size();  i++ )   {
        CRef<CNetBlastJobDescriptor>& descr = m_JobDescrs[i];
        if( ! descr->IsDeleted())  {
            descriptors.push_back(descr);
        }
    }
}


CRef<CNetBlastJobDescriptor> CNetBLASTUIDataSource::FindJobDescriptor(const string& rid)
{
    CFastMutexGuard guard(m_JobDescrMutex);

    CRef<CNetBlastJobDescriptor> result;
    for(  size_t i = 0;  i < m_JobDescrs.size();  i++ )   {
        CRef<CNetBlastJobDescriptor>& descr = m_JobDescrs[i];
        if(descr->GetRID() == rid)  {
            result = descr;
            break;
        }
    }
    return result;
}


static const wxChar* kBLAST_DBsPath = wxT("<home>/blast-dbs.asn");

bool CNetBLASTUIDataSource::x_NeedToRefreshCachedDBs()
{
    bool refresh = true;

    wxString path = CSysPath::ResolvePath(kBLAST_DBsPath);
    wxFileName fname(path);

    if (fname.FileExists()) {
        wxDateTime dtMod = fname.GetModificationTime();
        wxDateTime now = wxDateTime::Now();
        wxTimeSpan diff = now.Subtract(dtMod);
        if (diff.IsShorterThan(wxTimeSpan::Hours(8)))
            refresh = false;
    }
    return refresh;
}

// TODO rework error handling and messages
void CNetBLASTUIDataSource::x_ReadCachedDBs()
{
    CFastMutexGuard guard(m_DBLoadMutex);

    string msg;

    wxString path = CSysPath::ResolvePath(kBLAST_DBsPath);
    wxFileName fname(path);

    if (fname.FileExists()) {
        try {
            ESerialDataFormat fmts[] = {
                eSerial_AsnBinary,
                eSerial_AsnText
            };
            const size_t size_limit = sizeof(fmts) / sizeof(ESerialDataFormat);
            for (size_t i = 0;  i < size_limit;  ++i) {
                m_Databases.Reset(new CBlast4_get_databases_reply);
                try {
                    CNcbiIfstream istr(path.fn_str(), ios::in | ios::binary);
                    auto_ptr<CObjectIStream> is
                        (CObjectIStream::Open(fmts[i], istr));
                    *is >> *m_Databases;

                    break;
                }
                catch (CSerialException&) {
                    m_Databases.Reset();
                }
            }

            if ( !m_Databases ) {
                NCBI_THROW(CException, eUnknown,
                    "failed to read cached databases");
            }

            wxDateTime dtMod = fname.GetModificationTime();
            msg += "The previously cached version from ";
            msg += ToStdString(dtMod.Format(wxT("%B %d, %Y")));
            msg += " will be used.\nSome of the databases listed may not be available.";
            LOG_POST(Info << msg);
        } catch (CException&) {
            ::wxRemoveFile(path);
            m_Databases.Reset();
            msg += "The previously cached version could not be read.  "
                   "The BLAST4 service will be unavailable."; //TODO error reporting
        }
    } else {
        msg += "No cached databases were found.  "
               "The BLAST4 service will be unavailable."; //TODO
    }

    if ( !m_Databases ) {
        NCBI_THROW(CException, eUnknown, msg);
    }
}


/// Loads user fields describing BLAST DB hierarchy
void CNetBLASTUIDataSource::x_GetDBTree(TUserFields& fields)
{
    CStopWatch sw;
    sw.Start();
    wxString path = CSysPath::ResolvePath( wxT("<std>/etc/blastdb-spec/blast-db-tree.asn") );

    if (wxFileName::FileExists(path )) {
        try {
            CRef<CUser_field> field(new CUser_field());
            CNcbiIfstream istr(path.fn_str());
            istr >> MSerial_AsnText >> *field;

            /// flatten the hierarchy a bit - this is a triple-nested item
            ITERATE (CUser_field::TData::TFields, outer_it, field->GetData().GetFields()) {
                ITERATE (CUser_field::TData::TFields, inner_it, (*outer_it)->GetData().GetFields()) {
                    ITERATE (CUser_field::TData::TFields, it, (*inner_it)->GetData().GetFields()) {
                        fields.push_back(*it);
                    }
                }
            }
        }
        catch (CException& e) {
            // TODO error reporting
            LOG_POST(Error << "Error while loading blast-db-tree.asn: " << e.GetMsg());
        }
    }
    LOG_POST(Info << NCBI_CURRENT_FUNCTION << ": " << sw.Elapsed() << " seconds");
}


static const wxChar* kJobDescrPath = wxT("<home>/net_blast_jobs.asn");
static const char* kJobsTag = "Jobs";

void CNetBLASTUIDataSource::x_SaveJobDescriptors()
{
    LOG_POST(Info << "Net BLAST Data Source - saving job descriptors...");

    try {

        vector< CRef<CUser_object> >    jobs;

        for(  size_t i = 0;  i < m_JobDescrs.size();  i++  )  {
            try {
                CNetBlastJobDescriptor& descr = *m_JobDescrs[i];

                // do not save Descriptors marked as deleted, we preserve them
                // at run-time, but purge between application sessions
                if( ! descr.IsDeleted())    {
                    CRef<CUser_object> job_obj(descr.ToUserObject());
                    jobs.push_back(job_obj);
                }
            } catch(CException& e)  {
                LOG_POST(Error << "Saving Net BLAST jobs - exception while saving a job - " << e.GetMsg());
            }
        }

        CRef<CUser_object> container(new CUser_object());
        container->SetType().SetStr(kJobsTag);
        container->AddField(kJobsTag, jobs);

        wxString path = CSysPath::ResolvePath(kJobDescrPath);
        CNcbiOfstream ostr(path.fn_str());
        ostr << MSerial_AsnText << *container;

        LOG_POST(Info << "Net BLAST Data Source - finished saving job descriptors");
    } catch(CException& e)  {
        LOG_POST(Info << "Net BLAST Data Source - failed to save jobs - " << e.GetMsg());
    }
}


void CNetBLASTUIDataSource::x_LoadJobDescriptors()
{
    LOG_POST(Info << "Net BLAST Data Source - loading job descriptors...");

    wxString path = CSysPath::ResolvePath(kJobDescrPath);

    if (wxFileName::FileExists(path)) {
        try {
            CRef<CUser_object> container(new CUser_object());
            CNcbiIfstream istr(path.fn_str());
            istr >> MSerial_AsnText >> *container;

            const CUser_field& field = container->GetField(kJobsTag);
            const vector< CRef<CUser_object> >& jobs = field.GetData().GetObjects();

            m_JobDescrs.clear();
            for( size_t i = 0;  i < jobs.size();  i++ ) {
                const CUser_object& job_obj = *jobs[i];

                CRef<CNetBlastJobDescriptor> descr(new CNetBlastJobDescriptor(*this));
                descr->FromUserObject(job_obj);

                if(descr->GetState() != CNetBlastJobDescriptor::eInvalidState)    {
                    m_JobDescrs.push_back(descr);
                }
            }

            LOG_POST(Info << "Net BLAST Data Source - finished loading job descriptors");
        }
        catch (CException& e) {
            LOG_POST(Error << "Net BLAST Data Source - failed to load job descriptors" << e.GetMsg());
        }
    }
}


void  sPrepareDBDescr(string& db_desc)
{
    /// escape slashes in the description
    db_desc = NStr::Replace(db_desc, "/", "\\/");

    /// strip off external quotes
    string::size_type pos = db_desc.find_first_not_of("\"'");
    if (pos != string::npos) {
        db_desc.erase(0, pos);
    }
    pos = db_desc.find_last_not_of("\"'");
    if (pos != string::npos) {
        pos += 1;
        if (pos < db_desc.size()) {
            db_desc.erase(pos);
        }
    }
}


/// Fills sm_ProtDbMap and sm_NucDbMap maps.
/// The keys in the maps represent paths in the DB hierarchy.
void CNetBLASTUIDataSource::x_CategorizeDBs()
{
    TDbMap valid_db_names;

    typedef CBlast4_get_databases_reply::Tdata  TDBData;
    CBlast4_get_databases_reply::Tdata& db_data = m_Databases->Set();

    // iterate by all available Databases
    NON_CONST_ITERATE (TDBData, it, db_data) {
        const CBlast4_database_info& info = **it;
        const string& db_name = info.GetDatabase().GetName();

        // insert DB record into a temporary map
        valid_db_names.insert(TDbMap::value_type(db_name, *it));
    }

    NON_CONST_ITERATE (TDBData, it, db_data) {
        const CBlast4_database_info& info = **it;
        const string& db_name = info.GetDatabase().GetName();
        string db_desc = info.GetDescription();

        sPrepareDBDescr(db_desc);

        string name = db_name;
        string visible_name = db_name;
        if ( ! db_desc.empty()  &&  db_name.find(db_desc) == string::npos) {
            name += " (" + db_desc + ")";
            string db_desc_trimmed(db_desc);
            visible_name += " (" + db_desc_trimmed + ")";
        }

        /// 'nr' and 'nt' should never be subcategorized
        if (db_name == "nr"  ||  db_name == "nt") {
            bool prot = false;
            if (db_name == "nr") {
                prot = (info.GetDatabase().GetType() == eBlast4_residue_type_protein);
            }
            TDbMap& db = prot ? m_ProtDbMap : m_NucDbMap;
            db.insert(TDbMap::value_type(visible_name, *it));

            // TODO temp workaround
            vector<string>& dbs = prot ? m_Prot_DefMRU_DBs : m_Nuc_DefMRU_DBs;
            dbs.push_back(visible_name);
            continue;
        }
    }

    // use categorization information to build the final maps
    x_AddCategorizedDBs(valid_db_names);

    NON_CONST_ITERATE (TDbMap, it, valid_db_names) {
        it->second.Reset();
    }

    valid_db_names.clear();
}


void CNetBLASTUIDataSource::x_AddCategorizedDBs(const TDbMap& valid_db_names)
{
    CStopWatch sw; sw.Start();

    TUserFields fields;
    x_GetDBTree(fields);

    ITERATE (list< CConstRef<CUser_field> >, it, fields) {
        try {
            const CUser_field& f = **it;
            string label = f.GetLabel().GetStr();
            string db    = f.GetField("Db").GetData().GetStr();
            string desc  = f.GetField("Descr").GetData().GetStr();

            db = NStr::Replace(db, "//", "/");

            TDbMap::const_iterator dbname_iter = valid_db_names.find(db);
            if (dbname_iter == valid_db_names.end()) {
                // TODO
                LOG_POST(Warning << "dropping invalid BLAST database: " << db << " (" << desc << ")");
            } else{
                CRef<objects::CBlast4_database_info> info(dbname_iter->second);
                bool prot = (info->GetDatabase().GetType() == eBlast4_residue_type_protein);
                TDbMap& db = prot ? m_ProtDbMap : m_NucDbMap;
                db.insert(TDbMap::value_type(desc, info));
            }
        }
        catch (CException& e) {
            //TODO error reporting
            LOG_POST(Error << "error processing BLAST databases: " << e.GetMsg());
        }
    }
}


//TODO synchronize
void CNetBLASTUIDataSource::x_InitDatabases()
{
    x_ClearDatabases();

    /// check if we need to refresh this file
    bool refresh = x_NeedToRefreshCachedDBs();
    if ( !refresh ) {
        /// try and deserialize the saved portion
        x_ReadCachedDBs();
    }

    if ( !m_Databases ) {
        CIRef<IAppJob> job(new CNetBlastLoadDBListJob(this));
        CAppJobDispatcher& disp = CAppJobDispatcher::GetInstance();

        try {
            m_DBLoadJobId = disp.StartJob(*job, "ThreadPool", *this, 1, true);
        } catch(CAppJobException& e)  {
            ERR_POST("CNetBLASTUIDataSource::x_InitDatabases() - Failed to start BLAST DBs loading job");
            e.ReportAll();
        }
    } else {
        /// hack: categorize databases a bit
        x_CategorizeDBs();
    }

    if( ! m_Nuc_DefMRU_DBs.empty()) {
        m_Nuc_DefDB = m_Nuc_DefMRU_DBs[0];
    }
    if( ! m_Prot_DefMRU_DBs.empty()) {
        m_Prot_DefDB = m_Prot_DefMRU_DBs[0];
    }
}

void  CNetBLASTUIDataSource::x_SetDBLoadJobId(CAppJobDispatcher::TJobID job_id)
{
    CFastMutexGuard guard(m_DBLoadMutex);
    m_DBLoadJobId = job_id;    
}

void  CNetBLASTUIDataSource::x_SetDataBases(CRef<objects::CBlast4_get_databases_reply> dbs)
{
    CFastMutexGuard guard(m_DBLoadMutex);

    m_Databases = dbs;
    /// hack: categorize databases a bit
    x_CategorizeDBs();

    m_DBLoadJobId = 0;
}

void  CNetBLASTUIDataSource::x_WaitForDBLoadJob()
{
    LOG_POST(Info << "Wait-check for Blast BD background job finished.");
    for (unsigned i = 0; i < 2000; ++i) { // spin-lock, waiting for the job
        {
        CFastMutexGuard guard(m_DBLoadMutex);
        if (m_DBLoadJobId == 0) { // DB load job finished
            LOG_POST(Info << "Blast BD background job complete!");
            return;
        }
        }
        SleepMilliSec(100);
    }
    // we should never see this...
    LOG_POST(Error << "Blast BD background timeout expiration!");
}


void CNetBLASTUIDataSource::x_ClearDatabases()
{
    m_Nuc_DefMRU_DBs.clear();
    m_Prot_DefMRU_DBs.clear();

    m_Nuc_DefDB.clear();
    m_Prot_DefDB.clear();

    m_NucDbMap .clear();
    m_ProtDbMap.clear();

    m_Databases.Reset();
}


void CNetBLASTUIDataSource::x_OnJobDescrChanged(CNetBlastJobDescriptor& descr)
{
    // notify asynchronously
    x_NotifyObservers();
}


void CNetBLASTUIDataSource::StartRetrievingTask(TJobDescrVec& jobs)
{
    vector<string> RIDs;
    for(  size_t i = 0;  i < jobs.size();  i++) {
        CNetBlastJobDescriptor& descr = *jobs[i];
        RIDs.push_back(descr.GetRID());
    }

    CRef<CBlastSearchTask> task(new CBlastSearchTask(m_SrvLocator, *this, "Net BLAST Search"));
    task->Init_RetrieveRID(RIDs);

    x_StartTask(*task);
}


void CNetBLASTUIDataSource::StartMonitoringTask(TJobDescrVec& jobs)
{
    CRef<CBlastSearchTask> task(new CBlastSearchTask(m_SrvLocator, *this, "Net BLAST Search"));
    task->Init_Monitoring(jobs);

    x_StartTask(*task);
}


void CNetBLASTUIDataSource::DeleteJobDescriptors(TJobDescrVec& descriptors)
{
    LockUpdates();

    for(  size_t i = 0;  i < descriptors.size();  i++)  {
        CNetBlastJobDescriptor& descr = *descriptors[i];
        descr.MarkDeleted(true);
    }

    UnlockUpdates();
}


void CNetBLASTUIDataSource::OnExplore()
{
    if(m_SrvLocator)    {
        /// first show the view
        IViewManagerService* view_srv = m_SrvLocator->GetServiceByType<IViewManagerService>();
        CIRef<IView> view = view_srv->ShowSingletonView("Search View");

        // here - send command to the view
        CDataMiningView* dm_view = dynamic_cast<CDataMiningView*>(view.GetPointer());
        if(dm_view) {
            dm_view->SelectToolByName(NET_BLAST_DM_TOOL_NAME);
        }
    }
}


void CNetBLASTUIDataSource::OnLoadRIDs()
{
    /// TODO this is not a good solution, but simple
    IWorkbench* workbench = dynamic_cast<IWorkbench*>(m_SrvLocator);

    if(workbench)    {
        COpenDlgTask* task = new COpenDlgTask(workbench, NET_BLAST_LOADER_LABEL);

        CAppTaskService* task_srv = m_SrvLocator->GetServiceByType<CAppTaskService>();
        task_srv->AddTask(*task);
    }
}

void CNetBLASTUIDataSource::x_StartTask(CBlastSearchTask& task)
{
    CDataLoadingOptions options;
    options.Set_DecideLater();
    task.SetLoadingOptions(options);

    CAppTaskService* task_srv = m_SrvLocator->GetServiceByType<CAppTaskService>();
    task_srv->AddTask(task);
}


// start monitoring task for all jobs that are still in progress
void CNetBLASTUIDataSource::x_AutoStartMonitoring()
{
    TJobDescrVec monitor_jobs;
    {{
    CFastMutexGuard guard(m_JobDescrMutex);

    // get jobs that have "Submitted" status
    for(  size_t i = 0;  i < m_JobDescrs.size();  i++)  {
        CRef<CNetBlastJobDescriptor>& descr = m_JobDescrs[i];
        if(descr->GetState() == CNetBlastJobDescriptor::eSubmitted)  {
            monitor_jobs.push_back(descr);
        }
    }
    }}

    size_t n = monitor_jobs.size();
    if(n > 0)   {
        LOG_POST(Info << "Net BLAST start monitoring for " << n << " jobs");
        StartMonitoringTask(monitor_jobs);
    }
}


///////////////////////////////////////////////////////////////////////////////
/// CNetBLASTUIDataSourceType
CNetBLASTUIDataSourceType::CNetBLASTUIDataSourceType()
:   m_Descr( "Net BLAST Client Service", "", "", "", "NetBlastUI" )
{
   //wxFileArtProvider* provider = GetDefaultFileArtProvider();
   //TODO provider->RegisterFileAlias(kNetBLAST_DS_Icon, "net_blast_data_source.png");
}


const IUIObject& CNetBLASTUIDataSourceType::GetDescr()
{
    return m_Descr;
}


IUIDataSource* CNetBLASTUIDataSourceType::CreateDataSource()
{
    return new CNetBLASTUIDataSource(*this);
}


bool CNetBLASTUIDataSourceType::AutoCreateDefaultDataSource()
{
    return true; // create NCBI Net BLAST by default
}


string CNetBLASTUIDataSourceType::GetExtensionIdentifier() const
{
    static string ext_id("net_blast_data_source_type");
    return ext_id;
}


string CNetBLASTUIDataSourceType::GetExtensionLabel() const
{
    return m_Descr.GetLabel();
}


END_NCBI_SCOPE