bam_ui_data_source.cpp

Go to the documentation of this file.

/*  $Id: bam_ui_data_source.cpp 25315 2012-02-23 16:34:03Z wuliangs $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Liangshou Wu
 *
 */

#include <ncbi_pch.hpp>

#include <corelib/ncbiexec.hpp>
#include <corelib/ncbi_process.hpp>
#include <corelib/ncbi_system.hpp>
#include <corelib/ncbifile.hpp>

#include "bam_load_option_panel.hpp"
#include "bam_indexing_panel.hpp"
#include "bam_coverage_graph_panel.hpp"

#include <gui/packages/pkg_sequence/bam_ui_data_source.hpp>
#include <gui/core/app_tasks.hpp>
#include <gui/core/app_explorer_service.hpp>
#include <gui/core/loading_app_job.hpp>

#include <gui/framework/workbench.hpp>
#include <gui/framework/app_task_service.hpp>
#include <gui/framework/app_job_task.hpp>

#include <gui/widgets/wx/wx_utils.hpp>
#include <gui/widgets/wx/fileartprov.hpp>
#include <gui/widgets/wx/ui_command.hpp>
#include <gui/widgets/wx/message_box.hpp>

#include <gui/objects/LoaderDescriptor.hpp>
#include <gui/objutils/registry.hpp>
#include <gui/core/assm_info.hpp>
#include <gui/utils/extension_impl.hpp>
#include <gui/utils/system_path.hpp>

#include <wx/menu.h>

#include <objmgr/object_manager.hpp>
#include <objects/general/Object_id.hpp>
#include <objects/seq/Seq_annot.hpp>
#include <objects/seq/Annot_descr.hpp>
#include <objects/seq/Annotdesc.hpp>
#include <objects/seqres/Seq_graph.hpp>

#include <serial/objistr.hpp>
#include <serial/objostr.hpp>
#include <serial/serial.hpp>

#include <sra/data_loaders/bam/bamloader.hpp>
#include <sra/readers/bam/bamgraph.hpp>
#include <sra/readers/bam/bamread.hpp>
#include <sra/readers/ncbi_traces_path.hpp>
#include <objtools/readers/idmapper.hpp>

#include <sra/data_loaders/csra/csraloader.hpp>

#include <util/format_guess.hpp>
#include <corelib/ncbi_system.hpp>
#include <corelib/ncbiexec.hpp>

#include <sstream>

BEGIN_NCBI_SCOPE
USING_SCOPE(objects);

static const char* kLoaderDataType_File = "BAM file";
static const char* kLoaderDataType_SRZ = "BAM SRZ";
static const char* kLoaderDataType_CSra_File = "CSRA file";
static const char* kBamLoadOption = "BAM/CSRA files";
static const char* kBamPath = "path";
static const char* kBamFile = "files";
static const char* kCSraFile = "file";
static const char* kBamIndexFile = "index";
static const char* kTargetAssm = "context";
static const char* kSRZAccession = "SRZ";
static const int   kLoaderPriority = 98;


///////////////////////////////////////////////////////////////////////////////
/// CBamLoadingJob
class  CBamLoadingJob : public CDataLoadingAppJob
{
public:
    struct SBamFile
    {
        string m_BamFile;
        string m_IndexFile;
        string m_CoverageGraph;
    };

    typedef vector<SBamFile> TBamFiles;
    typedef set<string> TCSraFiles;
    typedef set<string> TSrzAccs;

    CBamLoadingJob(CProjectService* service);

    void SetSrzAccs(const TSrzAccs& accs) { m_SrzAccs = accs; }
    void SetBamFiles(const TBamFiles& files) { m_BamFiles = files; }
    void SetCSraFiles(const TCSraFiles& files) { m_CSraFiles = files; }
    void SetSamtoolsPath(const string& samtools) { m_SamtoolsPath = samtools; }
    void SetGraphBinSize(int size) { m_BinSize = size; }

    const string& GetTargetAssembly() const { return m_TargetAssm; }
    void SetTargetAssembly(const string& assm) { m_TargetAssm = assm; }

protected:
    virtual void x_CreateProjectItems();

private:
    TLoaderRef x_CreateDataLoader(SBamFile& bam_input);

    CRef<objects::CSeq_annot> x_CreateCoverageGraphs(const string& path,
        const string& bam_file, const string& index_file,
        const string& annot_name);

private:
    TSrzAccs  m_SrzAccs;        // SRZ accessions
    TBamFiles m_BamFiles;
    TCSraFiles m_CSraFiles;
    string  m_TargetAssm;
    string  m_SamtoolsPath;
    int     m_BinSize;
};


///////////////////////////////////////////////////////////////////////////////
/// CBamDSEvtHandler - wxEvtHandler-derived adapter for Bam data source.

class CBamDSEvtHandler : public wxEvtHandler
{
    DECLARE_EVENT_TABLE();
public:
    CBamDSEvtHandler(IWorkbench* workbench)
        :   m_Workbench(workbench)  {
    }
    void OnLoadBamFile(wxCommandEvent& event)
    {
        if(m_Workbench)    {
            COpenDlgTask* task = new COpenDlgTask(m_Workbench, kBamLoadOption);

            CAppTaskService* task_srv = m_Workbench->GetServiceByType<CAppTaskService>();
            task_srv->AddTask(*task);
        }
    }
protected:
    IWorkbench* m_Workbench;
};


BEGIN_EVENT_TABLE(CBamDSEvtHandler, wxEvtHandler)
    EVT_MENU(eCmdLoadBamFile, CBamDSEvtHandler::OnLoadBamFile)
END_EVENT_TABLE();


///////////////////////////////////////////////////////////////////////////////
/// CBamUIDataSource

static const char* kBAM_DS_Icon = "icon::bam_data_source";

CBamUIDataSource::CBamUIDataSource(CBamUIDataSourceType& type)
:   m_Type(&type),
    m_Descr("BAM", kBAM_DS_Icon),
    m_SrvLocator(NULL),
    m_Open(false)
{
}

CBamUIDataSource::~CBamUIDataSource()
{
}


string CBamUIDataSource::GetExtensionIdentifier() const
{
    return "bam_data_source";
}


string CBamUIDataSource::GetExtensionLabel() const
{
    return "Bam Data Source";
}


void CBamUIDataSource::SetServiceLocator(IServiceLocator* locator)
{
    m_SrvLocator = locator;
}


IExplorerItemCmdContributor::TContribution
    CBamUIDataSource::GetMenu(TItemRefVector& items,
                                   CAppExplorerService& app_service)
{
    IExplorerItemCmdContributor::TContribution contrib;

    /// this is not a good solution, but simple
    IWorkbench* workbench = dynamic_cast<IWorkbench*>(m_SrvLocator);

    if(items.size() != 1  ||  workbench == NULL)   {
        return contrib; // return empty object - nothin to contribute
    }

    CUICommandRegistry& cmd_reg = CUICommandRegistry::GetInstance();

    CExplorerItem* item = items[0].GetPointer();
    int type = app_service.GetItemType(*item);

    if(type == CAppExplorerService::eDataSource)    {
        CAppExplorerService::TDataSourceTreeItem* ds_item =
            dynamic_cast<CAppExplorerService::TDataSourceTreeItem*>(item);

        if(ds_item) {
            CIRef<IUIDataSource> ds = ds_item->GetData();
            CBamUIDataSource* bam_ds = dynamic_cast<CBamUIDataSource*>(ds.GetPointer());

            if(bam_ds)  {
                wxMenu* menu = new wxMenu;
                menu->Append(wxID_SEPARATOR, wxT("Top Actions"));
                cmd_reg.AppendMenuItem(*menu, eCmdLoadBamFile);

                contrib.first = menu;
                contrib.second = new CBamDSEvtHandler(workbench);
            }
        }
    }
    return contrib;
}


IUIDataSourceType& CBamUIDataSource::GetType() const
{
    return *m_Type;
}


const IUIObject& CBamUIDataSource::GetDescr()
{
    return m_Descr;
}


bool CBamUIDataSource::IsOpen()
{
    return m_Open;
}

//static const char* kCmdExtPoint = "scoped_objects::cmd_contributor";
static const char* kAppExpCmdExtPoint = "project_tree_view::context_menu::item_cmd_contributor";

bool CBamUIDataSource::Open()
{
    if (m_Open) {
        LOG_POST(Error << "CBamUIDataSource::Open(): "
            "attempt to open already open data source");
        return false;
    }

    CStopWatch sw;
    sw.Start();

    /// register itself as menu contributor
    CIRef<IExtensionRegistry> reg = CExtensionRegistry::GetInstance();
    reg->AddExtension(kAppExpCmdExtPoint, *this);

    m_Open = true;
    return true;
}


bool CBamUIDataSource::Close()
{
    if (m_Open) {
        /// remove itself from menu contribution points
        CIRef<IExtensionRegistry> reg = CExtensionRegistry::GetInstance();
        reg->RemoveExtension(kAppExpCmdExtPoint, *this);

        m_Open = false;
        return true;
    }

    return false;
}


void CBamUIDataSource::EditProperties()
{
    //TODO
}


IUIToolManager* CBamUIDataSource::GetLoadManager()
{
    // TODO may need to link the manager to this particular datasource
    return new CBamUILoadManager();
}


string CBamUIDataSource::AddDataLoader(const objects::CUser_object& obj)
{
    string loader_name = kEmptyStr;

    // figure out the loader name and data from object
    if (obj.GetType().IsStr()) {
        const string& loader_type = obj.GetType().GetStr();
        if (loader_type == kLoaderDataType_File) {
            string path, file, index, target_assm;
            if (x_GetBamInputs(obj, path, file, index, target_assm)) {
                // make sure the given BAM file exists
                if ( !CFile(path + file).Exists() ) {
                    string msg = "Can't find the bam file: " + path + file;
                    NCBI_THROW(CException, eUnknown, msg);
                }

                if ( !CFile(path + index).Exists() ) {
                    string msg = "Can't find the bam index file: " + path + index;
                    NCBI_THROW(CException, eUnknown, msg);
                }

                CBAMDataLoader::SLoaderParams params;
                params.m_DirPath = path;
                params.m_BamFiles.push_back(CBAMDataLoader::SBamFileName(file, index));

                if ( !target_assm.empty() ) {
                    //params.m_IdMapper.reset(new CIdMapperBuiltin(target_assm, false));
                    string mapper_str;
                    CAssemblyInfo::GetIdMapperString(target_assm, mapper_str);
                    std::stringstream mapper_stream(mapper_str);
                    params.m_IdMapper.reset(new CIdMapperConfig(mapper_stream, target_assm));
                }
                CRef<objects::CObjectManager> obj_mgr = CObjectManager::GetInstance();
                CBAMDataLoader::TRegisterLoaderInfo info =
                    CBAMDataLoader::RegisterInObjectManager(*obj_mgr, params,
                    CObjectManager::eNonDefault, kLoaderPriority);
                loader_name = info.GetLoader()->GetName();
            }
        } else if (loader_type == kLoaderDataType_CSra_File) {
            string file;
            if (x_GetCSraInputs(obj, file)) {
                // make sure the given BAM file exists
                if ( !CFile(file).Exists() ) {
                    string msg = "Can't find the CSRA file: " + file;
                    NCBI_THROW(CException, eUnknown, msg);
                }

                CRef<objects::CObjectManager> obj_mgr = CObjectManager::GetInstance();
                CCSRADataLoader::TRegisterLoaderInfo info =
                    CCSRADataLoader::RegisterInObjectManager
                        (*obj_mgr, "", file,
                         CObjectManager::eNonDefault, kLoaderPriority);
                loader_name = info.GetLoader()->GetName();
            }
        } else if (loader_type == kLoaderDataType_SRZ) {
            string srz;
            if (x_GetSRZAccession(obj, srz)) {
                CRef<objects::CObjectManager> obj_mgr = CObjectManager::GetInstance();
                try {
                    CBAMDataLoader::TRegisterLoaderInfo info =
                        CBAMDataLoader::RegisterInObjectManager(*obj_mgr, srz,
                        CObjectManager::eNonDefault, kLoaderPriority);
                    loader_name = info.GetLoader()->GetName();
                } catch (CException&) {
                    objects::CCSRADataLoader::TRegisterLoaderInfo info;
                    info = objects::CCSRADataLoader::RegisterInObjectManager(*obj_mgr, srz,
                        objects::CObjectManager::eNonDefault);
                    loader_name = info.GetLoader()->GetName();
                }
            }
        }
    }

    return loader_name;
}


string CBamUIDataSource::GetLoaderName(const objects::CUser_object& obj) const
{
    string loader_name;

    if (obj.GetType().IsStr()) {
        const string& loader_type = obj.GetType().GetStr();
        if (loader_type == kLoaderDataType_File) {
            string path, file, index, target_assm;
            if (x_GetBamInputs(obj, path, file, index, target_assm)) {
                vector<CBAMDataLoader::SBamFileName> bam_files;
                bam_files.push_back(CBAMDataLoader::SBamFileName(file, index));
                loader_name =
                    CBAMDataLoader::GetLoaderNameFromArgs(path, bam_files);
            }
        } else if (loader_type == kLoaderDataType_SRZ) {
            string srz;
            if (x_GetSRZAccession(obj, srz)) {
                loader_name = CBAMDataLoader::GetLoaderNameFromArgs(srz);
            }
        } 
    }

    return loader_name;
}


bool CBamUIDataSource::x_GetBamInputs(const objects::CUser_object& obj,
                                      string& path, string& file,
                                      string& idx, string& target_assm) const
{
    CConstRef<CUser_field> field_path = obj.GetFieldRef(kBamPath); 
    CConstRef<CUser_field> field_file = obj.GetFieldRef(kBamFile);
    CConstRef<CUser_field> field_index = obj.GetFieldRef(kBamIndexFile);
    CConstRef<CUser_field> field_target_assm = obj.GetFieldRef(kTargetAssm); 
    
    if (field_path  &&  field_file  &&  field_index  &&  field_target_assm) {
        path = field_path->GetData().GetStr();
        file = field_file->GetData().GetStr();
        idx = field_index->GetData().GetStr();
        target_assm = field_target_assm->GetData().GetStr();
        
        return true;
    }

    return false;
}


bool CBamUIDataSource::x_GetSRZAccession(const objects::CUser_object& obj,
                                         string& srz) const
{
    CConstRef<CUser_field> field_srz = obj.GetFieldRef(kSRZAccession);
    if (field_srz) {
        srz = field_srz->GetData().GetStr();
        return true;
    }
    return false;
}


bool CBamUIDataSource::x_GetCSraInputs(const objects::CUser_object& obj,
                                       string& file) const
{
    CConstRef<CUser_field> field_file = obj.GetFieldRef(kCSraFile);
    
    if (field_file) {
        file = field_file->GetData().GetStr();
        
        return true;
    }

    return false;
}


///////////////////////////////////////////////////////////////////////////////
/// CBamUIDataSourceType
CBamUIDataSourceType::CBamUIDataSourceType()
    :   m_Descr("Bam File data Loader", "")
{
   wxFileArtProvider* provider = GetDefaultFileArtProvider();
   provider->RegisterFileAlias(ToWxString(kBAM_DS_Icon),
                               wxT("bam_data_source.png"));
}


const IUIObject& CBamUIDataSourceType::GetDescr()
{
    return m_Descr;
}


IUIDataSource* CBamUIDataSourceType::CreateDataSource()
{
    return new CBamUIDataSource(*this);
}


bool CBamUIDataSourceType::AutoCreateDefaultDataSource()
{
    return true; // we want to create default "Bam" datasource
}


string CBamUIDataSourceType::GetExtensionIdentifier() const
{
    static string ext_id("bam_data_source_type");
    return ext_id;
}


string CBamUIDataSourceType::GetExtensionLabel() const
{
    return m_Descr.GetLabel();
}



///////////////////////////////////////////////////////////////////////////////
/// CBamUILoadManager
CBamUILoadManager::CBamUILoadManager()
:   m_SrvLocator(NULL),
    m_ParentWindow(NULL),
    m_Descriptor(kBamLoadOption, ""),
    m_State(eInvalid),
    m_AssmWizard(CGCAssemblySearchWizard::eIdMapping),
    m_OptionPanel(NULL),
    m_IndexingPanel(NULL),
    m_GraphPanel(NULL),
    m_ProjectSelPanel(NULL)
{
    m_ProjectParams.m_EnableDecideLater = false;
}


void CBamUILoadManager::SetServiceLocator(IServiceLocator* srv_locator)
{
    m_SrvLocator = srv_locator;
}


void CBamUILoadManager::SetParentWindow(wxWindow* parent)
{
    m_ParentWindow = parent;
    m_AssmWizard.SetParentWindow(parent);
}


const IUIObject& CBamUILoadManager::GetDescriptor() const
{
    return m_Descriptor;
}


void CBamUILoadManager::InitUI()
{
    m_State = eSelectBam;
    m_AssmWizard.InitUI();
}


void CBamUILoadManager::CleanUI()
{
    m_State = eInvalid;
    if(m_OptionPanel)   {
        m_SavedInput = m_OptionPanel->GetInput();
        m_OptionPanel = NULL; // window is destroyed by the system
    }
    if (m_IndexingPanel) {
        m_SamtoolsPath = m_IndexingPanel->GetSamtoolsPath();
        m_IndexingPanel = NULL;
    }
    m_GraphPanel = NULL;
    m_ProjectSelPanel = NULL;
    m_AssmWizard.CleanUI();
}


wxPanel* CBamUILoadManager::GetCurrentPanel()
{
    if (m_State == eSelectBam) {
        if (m_OptionPanel == NULL) {
            m_OptionPanel = new CBamLoadOptionPanel(m_ParentWindow);
            m_OptionPanel->SetInput(m_SavedInput);
        }
        return m_OptionPanel;
    } else if (m_State == eBamIndex) {
        if (m_IndexingPanel == NULL) {
            m_IndexingPanel = new CBamIndexingPanel(m_ParentWindow);
            m_IndexingPanel->SetSamtoolsPath(m_SamtoolsPath);
            m_IndexingPanel->SetBamFilePath(m_OptionPanel->GetBamFilePath());
        }
        return m_IndexingPanel;
    } else if (m_State == eCoverageGraph) {
        if (m_GraphPanel == NULL) {
            m_GraphPanel = new CBamCoverageGraphPanel(m_ParentWindow);
            m_GraphPanel->SetBamFiles(m_OptionPanel->GetBamFiles());
        }
        return m_GraphPanel;
    } else if (m_State == eIdMapping) {
        return m_AssmWizard.GetCurrentPanel();
    } else if (m_State == eSelectProject) {
        if(m_ProjectSelPanel == NULL)   {
            CIRef<CProjectService> srv = m_SrvLocator->GetServiceByType<CProjectService>();
            m_ProjectSelPanel = new CProjectSelectorPanel(m_ParentWindow);
            m_ProjectSelPanel->SetProjectService(srv);
            m_ProjectSelPanel->SetParams(m_ProjectParams);
            m_ProjectSelPanel->TransferDataToWindow();
        }
        return m_ProjectSelPanel;
    }
    return NULL;
}


bool CBamUILoadManager::CanDo(EAction action)
{
    switch(m_State) {
    case eSelectBam:
        return action == eNext;
    case eBamIndex:
    case eCoverageGraph:
    case eIdMapping:
    case eSelectProject:
        return action == eBack  ||  action == eNext;
    case eCompleted:
        return false; // nothing left to do
    default:
        _ASSERT(false);
        return false;
    }
}


bool CBamUILoadManager::IsFinalState()
{
    return m_State == eSelectProject;
}


bool CBamUILoadManager::IsCompletedState()
{
    return m_State == eCompleted;
}


bool CBamUILoadManager::DoTransition(EAction action, wxPanel* currPanel)
{
    if (m_State == eSelectBam  &&  action == eNext) {
        if (m_OptionPanel->IsInputValid()) {
            if (m_OptionPanel->AllSRZ()) {
                m_State = eSelectProject;
                return true;
            } else {
                if (m_OptionPanel->HasIndexFile()) {
                    m_State = eCoverageGraph;
                } else {
                    m_State = eBamIndex;
                }
                return true;
            }
        }
        return false;
    } else if (m_State == eBamIndex) {
        if (action == eBack) {
            m_State = eSelectBam;
            return true;
        } else if (action == eNext) {
            if (m_IndexingPanel->IsInputValid()) {
                m_State = eCoverageGraph;
                return true;
            }
            return false;
        }
    } else if (m_State == eCoverageGraph) {
        if (action == eBack) {
            if (m_OptionPanel->HasIndexFile()) {
                m_State = eSelectBam;
            } else {
                m_State = eBamIndex;
            }
            return true;
        } else if (action == eNext) {
            if (m_GraphPanel->IsInputValid()) {
                m_State = eIdMapping;
                return true;
            }
            return false;
        }
    } else if (m_State == eIdMapping) {
        if (action == eSkip) {
            m_State = eSelectProject;
            m_AssmAcc.clear();
            return true;
        } else if (action == eBack) {
            if (m_AssmWizard.IsInitialState()) {
                m_State = eCoverageGraph;
                return true;
            } else {
                return m_AssmWizard.DoTransition(action, currPanel);
            }
        } else if (action == eNext) {
            if ( !m_AssmWizard.CanDo(action) ) {
                vector<string> assm_accs = m_AssmWizard.GetAssmAccs();
                if ( !assm_accs.empty() ) {
                    m_AssmAcc = assm_accs[0];
                    m_State = eSelectProject;
                    return true;
                }
                return false;
            } else {
                return m_AssmWizard.DoTransition(action, currPanel);
            }
        }
    } else if (m_State == eSelectProject) {
        if (action == eBack) {
            if (m_OptionPanel->AllSRZ()) {
                m_State = eSelectBam;
            } else {
                m_State = eIdMapping;
            }
            return true;
        } else if (action == eNext) {
            if(m_ProjectSelPanel->TransferDataFromWindow()) {
                m_State = eCompleted;
                return true;
            }
            return false;
        }
    }
    _ASSERT(false);
    return false;
}


IAppTask* CBamUILoadManager::GetTask()
{
    // create loading Job
    m_ProjectSelPanel->GetParams(m_ProjectParams);

    CIRef<CProjectService> srv = m_SrvLocator->GetServiceByType<CProjectService>();
    CBamLoadingJob* job = new CBamLoadingJob(srv.GetPointer());
    // create a wrapper task
    auto_ptr<CDataLoadingAppTask> task(new CDataLoadingAppTask(*job));
    task->SetReportErrors(false); // we handle errors ourselves

    CDataLoadingOptions& options = job->GetOptions();
    m_ProjectParams.ToLoadingOptions(options);

    CBamLoadingJob::TBamFiles bam_files;
    ITERATE (CBamLoadOptionPanel::TBamFiles, iter, m_OptionPanel->GetBamFiles()) {
        CBamLoadingJob::SBamFile bam_input;
        bam_input.m_BamFile = *iter;
        if (CFile(bam_input.m_BamFile + ".bai").Exists()) {
            string bam_file, ext;
            CFile::SplitPath(bam_input.m_BamFile, NULL, &bam_file, &ext);
            bam_input.m_IndexFile = bam_file + ext + ".bai";
        }
        if (m_GraphPanel) {
            bam_input.m_CoverageGraph = m_GraphPanel->GetGraphFile(*iter);
        }
        bam_files.push_back(bam_input);
    }

    job->SetSrzAccs(m_OptionPanel->GetSrzAccs());
    job->SetBamFiles(bam_files);
    job->SetCSraFiles(m_OptionPanel->GetCSraFiles());
    if (m_IndexingPanel) {
        job->SetSamtoolsPath(m_IndexingPanel->GetSamtoolsPath());
    }
    if (m_GraphPanel) {
        job->SetGraphBinSize(m_GraphPanel->GetGraphBinSize());
    }

    job->SetTargetAssembly(m_AssmAcc);

    return task.release();
}


void CBamUILoadManager::SetRegistryPath(const string& path)
{
    m_RegPath = path; // store for later use
    m_AssmWizard.SetRegistryPath(path);
}


static const char* kBamInputTag = "BamInput";
static const char* kSamtoolsPathTag = "SamtoolsPath";
static const char* kProjectParamsTag = "ProjectParams";


void CBamUILoadManager::SaveSettings() const
{
    if ( !m_RegPath.empty() ) {
        CGuiRegistry& gui_reg = CGuiRegistry::GetInstance();
        CGuiRegistry::TReadWriteView view = gui_reg.GetReadWriteView(m_RegPath);

        /// remember the selected Format (only if m_OptionPanel exists)
        if (m_OptionPanel) {
            m_SavedInput = m_OptionPanel->GetInput();
        }
        if (m_SavedInput.size() < 10000) {
            // do not save large texts
            view.Set(kBamInputTag, NStr::URLEncode(m_SavedInput));
        }

        if (m_IndexingPanel) {
            m_SamtoolsPath = m_IndexingPanel->GetSamtoolsPath();
        }

        if ( !m_SamtoolsPath.empty() ) {
            view.Set(kSamtoolsPathTag, m_SamtoolsPath);
        }
        /// save Project Panel settings
        m_ProjectParams.SaveSettings(view, kProjectParamsTag);
        m_AssmWizard.SaveSettings();
    }
}


void CBamUILoadManager::LoadSettings()
{
    if( ! m_RegPath.empty())   {
        CGuiRegistry& gui_reg = CGuiRegistry::GetInstance();
        CGuiRegistry::TReadView view = gui_reg.GetReadView(m_RegPath);

        m_SavedInput = view.GetString(kBamInputTag, kEmptyStr);
        NStr::URLDecodeInPlace(m_SavedInput);

        m_SamtoolsPath = view.GetString(kSamtoolsPathTag, kEmptyStr);

        /// load Project Panel settings
        m_ProjectParams.LoadSettings(view, kProjectParamsTag);
        m_AssmWizard.LoadSettings();
    }
}


/// CBamLoadingJob

CBamLoadingJob::CBamLoadingJob(CProjectService* service)
    : CDataLoadingAppJob(service)
    , m_BinSize(1000)
{
    CFastMutexGuard lock(m_Mutex);
    m_Descr = "Loading BAM file";
}

inline string s_QuotedPath(const string& path)
{
#if defined __WXMSW__
    return "\"" + path + "\"";
#else
    return path;
#endif
}

string s_GetTempSortedBamFile(const string& sorted_file, int file_i)
{
    string i_str = NStr::IntToString(file_i);
    if (i_str.length() < 4) {
        i_str.insert((size_t)0, size_t(4 - i_str.length()), '0');
    }
    return sorted_file + "." + i_str + ".bam";
}



string s_GetAnnotName(const CSeq_annot& annot)
{
    if (annot.IsSetDesc()) {
        ITERATE (CSeq_annot::TDesc::Tdata, iter, annot.GetDesc().Get()) {
            if ((*iter)->IsName()) {
                return (*iter)->GetName();
            }
        }
    }

    return "";
}


CRef<CSeq_annot> s_LoadSeqAnnot(const string& file, const string& annot_name)
{
    // Currently, only ASN.1, Binary ASN.1 or XML files are acceptible
    CNcbiIfstream istr(file.c_str(), ios::binary);
    ESerialDataFormat sfmt = eSerial_None;
    CFormatGuess fg;
    CFormatGuess::EFormat f = fg.Format(istr);
    switch (f) {
        case CFormatGuess::eBinaryASN:
            sfmt = eSerial_AsnBinary;
            break;
        case CFormatGuess::eTextASN:
            sfmt = eSerial_AsnText;
            break;
        case CFormatGuess::eXml:
            sfmt = eSerial_Xml;
            break;
        default:
            {{
                string msg("File format could not be determined:\n");
                msg += file;
                LOG_POST(Info << msg);
            }}
            break;
    }
    if (sfmt == eSerial_None) {
        string msg = "The graph file format is not supported!";
        msg += "\nGraph file: " + file;
        NCBI_THROW(CException, eUnknown, msg);
    }

    istr.seekg(0);
    auto_ptr<CObjectIStream> obj_str(CObjectIStream::Open(sfmt, istr));
    CRef<CSeq_annot> annot(new CSeq_annot);
    try {
        // We only consider seq-annot
        *obj_str >> *annot;
    } catch (CException& e) {
        string msg = "Can't deserialize the graph file. A seq-annot is expected. ";
        msg += "Error: " + e.GetMsg();
        msg += "\nGraph file: " + file;
        NCBI_THROW(CException, eUnknown, msg);
    }

    if ( !annot->IsSetData()  ||  !annot->IsGraph()) {
        string msg = "The input graph file doesn't cotain a graph!";
        msg += "\nGraph file: " + file;
        NCBI_THROW(CException, eUnknown, msg);
    }

    if ( s_GetAnnotName(*annot) != annot_name) {
        string msg = "The graph annotatin name (" + s_GetAnnotName(*annot);
        msg += ") needs to match with BAM file: " + annot_name;
        msg += "\nGraph file: " + file;
        NCBI_THROW(CException, eUnknown, msg);
    }

    return annot;
}


void CBamLoadingJob::x_CreateProjectItems()
{
    NON_CONST_ITERATE (TBamFiles, iter, m_BamFiles) {
        TLoaderRef bam_loader = x_CreateDataLoader(*iter);
        if (x_IsCanceled() ) {
            return;
        }

        if ( !bam_loader ) continue;

        // skip the one that doesn't have an index file
        if (iter->m_IndexFile.empty()) continue;

        const string& bam_file = iter->m_BamFile;
        string path, file, ext;
        CFile::SplitPath(bam_file, &path, &file, &ext);
        CRef<CSeq_annot> annot;

        LOG_POST(Info << "Create Project Items for BAM file: " << bam_file);

        if ( !CFile(iter->m_CoverageGraph).Exists() ) {
            // use Bam2Graph app to create a coverage graph on the fly
            // prepare parameters
            x_SetStatusText( "Creating BAM coverage graphs..." );
            annot = x_CreateCoverageGraphs(path, file + ext, iter->m_IndexFile, file);
            if ( annot  &&  !iter->m_CoverageGraph.empty() ) {
                // save to a file
                auto_ptr<CObjectOStream> 
                    out(CObjectOStream::Open(eSerial_AsnBinary, iter->m_CoverageGraph));
                *out << *annot;
            }
        } else {
            annot = s_LoadSeqAnnot(iter->m_CoverageGraph, file);
        }

        if (!annot) {
            return;
        }

        CRef<CProjectItem> item(new CProjectItem());
        item->SetObject(*annot);
        string label = item->GetLabelByData(m_Scope.GetPointer());
        if (label.empty()) {
            label = file;
        }

        if (label.find("coverage") == string::npos) {
            label += " (coverage graph)";
        }

        item->SetLabel(label);
        m_Items.push_back(item);

        m_Loaders.insert(TLoaders::value_type(item.GetPointer(), bam_loader));
    }

    // add data loader from SRZ accessions
    ITERATE (TSrzAccs, iter, m_SrzAccs) {
        if (x_IsCanceled()) {
            return;
        }

        CRef<CLoaderDescriptor> loader(new CLoaderDescriptor());
        CBamUIDataSourceType data_source_type;

        loader->SetLabel(*iter);
        loader->SetPriority(kLoaderPriority);
        loader->SetLoader_type(data_source_type.GetExtensionIdentifier());

        CRef<CLoaderDescriptor::TData> bam_data(new CLoaderDescriptor::TData);
        bam_data->SetType().SetStr(kLoaderDataType_SRZ);
        bam_data->AddField(kSRZAccession, *iter);

        loader->SetData(*bam_data);
        m_Loaders.insert(TLoaders::value_type(
            (const ncbi::objects::CProjectItem*)NULL, loader));
    }

    // add data loader from CSRA files
    ITERATE (TCSraFiles, iter, m_CSraFiles) {
        if (x_IsCanceled()) {
            return;
        }

        CRef<CLoaderDescriptor> loader(new CLoaderDescriptor());
        CBamUIDataSourceType data_source_type;

        loader->SetLabel(*iter);
        loader->SetPriority(kLoaderPriority);
        loader->SetLoader_type(data_source_type.GetExtensionIdentifier());

        CRef<CLoaderDescriptor::TData> bam_data(new CLoaderDescriptor::TData);
        bam_data->SetType().SetStr(kLoaderDataType_CSra_File);
        bam_data->AddField(kCSraFile, *iter);

        loader->SetData(*bam_data);
        m_Loaders.insert(TLoaders::value_type(
            (const ncbi::objects::CProjectItem*)NULL, loader));
    }
}


CBamLoadingJob::TLoaderRef
CBamLoadingJob::x_CreateDataLoader(CBamLoadingJob::SBamFile& bam_input)
{
    TLoaderRef bam_loader;
    const string& bam_file = bam_input.m_BamFile;
    if (bam_file.empty()) {
        LOG_POST(Error << "CBamLoadingJob::x_CreateDataLoader(): "
            "Invalid (empty) bam file.");
        return bam_loader;
    }

    // bam file
    string path, file_name, ext;
    CDirEntry::SplitPath(bam_file, &path, &file_name, &ext);

    if (path.empty()  ||  file_name.empty()) {
        return bam_loader;
    }

    if (bam_input.m_IndexFile.empty()) {
        LOG_POST(Info << "Generating Bam index file for " << bam_file);
        x_SetStatusText("Generating Bam index file");

        // need to create an index file
        if (m_SamtoolsPath.empty()) {
            return bam_loader;
        }

        string index_file = bam_file + ".bai";

        CExec::CResult result = CExec::SpawnL(CExec::eNoWait,
            m_SamtoolsPath.c_str(), "index",
            s_QuotedPath(bam_file).c_str(),
            s_QuotedPath(index_file).c_str(), NULL);

        // wait here until it finishes
        CProcess proc(result.GetProcessHandle());
        while(proc.IsAlive()) {
            if (x_IsCanceled()  &&  proc.Kill()) {
                // need clean up the possible generated index file?
                CFile t_file(index_file);
                if (t_file.Exists()) {
                    t_file.Remove();
                }
                return bam_loader;
            }
            proc.Wait(500);
        }

        // if failed, try sorting the Bam file, and do it again
        if ( !CFile(index_file).Exists() ) {
            LOG_POST(Warning << "Maybe the input Bam file is not sorted. "
                << "Try sorting it and do indexing again ...");

            x_SetStatusText("Sorting Bam file");
            // sorting BAM file
            file_name += ".sorted";
            string sorted_file = path + file_name;

            // wait here until it finishes
            result = CExec::SpawnL(CExec::eNoWait,
                m_SamtoolsPath.c_str(), "sort",
                s_QuotedPath(bam_file).c_str(),
                s_QuotedPath(sorted_file).c_str(), NULL);
            CProcess sort_proc(result.GetProcessHandle());

            while(sort_proc.IsAlive()) {
                if (x_IsCanceled()  &&  sort_proc.Kill()) {
                    // need clean up the possible generated index file?
                    int file_i = 0;
                    string t_file = s_GetTempSortedBamFile(sorted_file, file_i);
                    while (file_i < 10000  &&  CFile(t_file).Exists()) {
                        CFile(t_file).Remove();
                        ++file_i;
                        t_file = s_GetTempSortedBamFile(sorted_file, file_i);
                    }
                    return bam_loader;
                }
                sort_proc.Wait(2000);
            }

            // samtools automatically append '.bam' to the given sorted file name.
            // so we need to adjust it 
            sorted_file += ".bam";
            if (CFile(sorted_file).Exists()) {
                x_SetStatusText("Generating Bam index file");

                // use the sorted Bam file instead of the original user-provided one
                index_file = sorted_file + ".bai";
                result = CExec::SpawnL(CExec::eNoWait,
                    m_SamtoolsPath.c_str(), "index",
                    s_QuotedPath(sorted_file).c_str(),
                    s_QuotedPath(index_file).c_str(), NULL);

                // wait here until it finishes
                CProcess index_proc(result.GetProcessHandle());
                while(index_proc.IsAlive()) {
                    if (x_IsCanceled()  &&  index_proc.Kill()) {
                        // need clean up the possible generated index file
                        CFile t_file(index_file);
                        if (t_file.Exists()) {
                            t_file.Remove();
                        }
                        return bam_loader;
                    }
                    index_proc.Wait(500);
                }

                if (CFile(index_file).Exists()) {
                    bam_input.m_BamFile = sorted_file;
                } else {
                    string msg = "Failed to generate index file for " + bam_file;
                    NCBI_THROW(CException, eUnknown, msg);
                }
            } else {
                string msg = "Failed to sort the Bam file for " + bam_file;
                NCBI_THROW(CException, eUnknown, msg);
            }
        }

        bam_input.m_IndexFile = file_name + ext + ".bai";
    }

    x_SetStatusText("Creating Bam data loader");
    file_name += ext;
    bam_loader.Reset(new CLoaderDescriptor());
    CBamUIDataSourceType data_source_type;

    bam_loader->SetLabel(file_name);
    bam_loader->SetPriority(kLoaderPriority);
    bam_loader->SetLoader_type(data_source_type.GetExtensionIdentifier());

    CRef<CLoaderDescriptor::TData> bam_data(new CLoaderDescriptor::TData);
    bam_data->SetType().SetStr(kLoaderDataType_File);

    bam_data->AddField(kBamFile, file_name);
    bam_data->AddField(kBamPath, path);
    bam_data->AddField(kBamIndexFile, file_name + ".bai");
    bam_data->AddField(kTargetAssm, GetTargetAssembly());

    bam_loader->SetData(*bam_data);
    return bam_loader;
}


static
CRef<CSeq_id> s_GetRefSeq_id(const string& id_str)
{
    CRef<CSeq_id> seq_id;
    try {
        seq_id = new CSeq_id(id_str);
    } catch ( CException&) {
        // ignored
    }

    if ( !seq_id && id_str.find('|') != NPOS ) {
        try {
            CBioseq::TId ids;
            CSeq_id::ParseIDs(ids, id_str);
            if ( !ids.empty() ) {
                seq_id = *ids.begin();
            }
        } catch ( CException& /*ignored*/ ) {
        }
    }

    if ( !seq_id || (seq_id->IsGi() && seq_id->GetGi() < 1000) ) {
        seq_id = new CSeq_id(CSeq_id::e_Local, id_str);
    }
    return seq_id;
}


CRef<CSeq_annot>
CBamLoadingJob::x_CreateCoverageGraphs(const string& path,
                                       const string& bam_file,
                                       const string& index_file,
                                       const string& annot_name)
{
    CRef<CSeq_annot> annot;
    CBamMgr mgr;

    // Get a list reference sequence ids
    vector<string> ref_ids;
    CBamDb db(mgr, path + bam_file, path + index_file);
    for (CBamRefSeqIterator sit(db); sit; ++sit) {
        if (x_IsCanceled()) {
            return annot;
        }
        ref_ids.push_back(sit.GetRefSeqId());
    }

    int i = 0;
    bool need_id_mapping = false;
    ITERATE (vector<string>, ref_it, ref_ids) {
        if (x_IsCanceled()) {
            annot.Reset(NULL);
            break;
        }
        string msg = "Creating BAM coverage graphs " +
            NStr::IntToString(i++) + "/" +
            NStr::SizetToString(ref_ids.size()) + " ...";
        x_SetStatusText(msg);

        CBam2Seq_graph cvt;
        cvt.SetRefLabel(*ref_it);
        CRef<CSeq_id> seq_id = s_GetRefSeq_id(*ref_it);
        if (seq_id->IsLocal()) {
            need_id_mapping = true;
        }
        cvt.SetRefId(*seq_id);
        cvt.SetAnnotName(annot_name);
        cvt.SetGraphTitle(annot_name + " coverage graph");
        cvt.SetGraphValueType(cvt.eGraphValueTyps_int);
        cvt.SetGraphBinSize(m_BinSize);
        //cvt.SetOutlierMax(args["max"].AsDouble());
        CRef<CSeq_annot> annot_t = cvt.MakeSeq_annot(db, path + bam_file);

        if (annot_t) {
            if (annot) {
                // move all graphs to one annotation since
                // they all share the same name and title
                CSeq_annot::TData::TGraph& target = annot->SetData().SetGraph();
                CSeq_annot::TData::TGraph& src = annot_t->SetData().SetGraph();
                std::copy(src.begin(), src.end(), back_inserter(target));
            } else {
                annot = annot_t;
            }
        }
    }

    if (annot) {
        annot->SetTitleDesc(annot_name + " coverage graph");
        if (need_id_mapping  &&  !m_TargetAssm.empty() ) {
            //CIdMapperBuiltin mapper(m_TargetAssm);
            string mapper_str;
            CAssemblyInfo::GetIdMapperString(m_TargetAssm, mapper_str);
            std::stringstream mapper_stream(mapper_str);
            CIdMapperConfig mapper(mapper_stream, m_TargetAssm);
            mapper.MapObject(*annot);
        }
    }
    
    return annot;
}


END_NCBI_SCOPE