paged_text.h

Go to the documentation of this file.

/*
 * nvbio
 * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of the NVIDIA CORPORATION nor the
 *      names of its contributors may be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#pragma once

#include <nvbio/basic/vector.h>
#include <nvbio/basic/packedstream.h>
#include <nvbio/basic/primitives.h>
#include <nvbio/basic/popcount.h>
#include <nvbio/basic/algorithms.h>
#include <nvbio/basic/exceptions.h>
#include <nvbio/basic/timer.h>
#include <nvbio/basic/cast_iterator.h>
#include <thrust/adjacent_difference.h>
#include <thrust/iterator/counting_iterator.h>
#include <thrust/iterator/permutation_iterator.h>
#include <thrust/iterator/transform_iterator.h>
#include <stack>
#include <numeric>

namespace nvbio {

void build_buckets(const uint64 key_range, const uint32 n_keys, const uint64* keys, const uint32 bucket_size, nvbio::vector<host_tag,uint32>& buckets, const bool upper = true);

template <uint32 SYMBOL_SIZE_T, bool BIG_ENDIAN_T>
struct PagedText
{
    typedef uint64                                                           word_type;
    typedef PackedStream<const word_type*,uint8,SYMBOL_SIZE_T,BIG_ENDIAN_T>  const_packed_page_type;
    typedef PackedStream<      word_type*,uint8,SYMBOL_SIZE_T,BIG_ENDIAN_T>        packed_page_type;

    static const uint32 SYMBOL_SIZE      = SYMBOL_SIZE_T;
    static const uint32 SYMBOL_COUNT     = 1u << SYMBOL_SIZE;
    static const uint32 WORD_SIZE        = (8u*sizeof(word_type));
    static const uint32 SYMBOLS_PER_WORD = WORD_SIZE / SYMBOL_SIZE;
    static const uint32 BIG_ENDIAN       = BIG_ENDIAN_T;

    static const uint32 LOG_BUCKET_SIZE = 20u;
    static const uint32     BUCKET_SIZE = 1u << LOG_BUCKET_SIZE;

    PagedText(
        const uint32 page_size    = 512 * 1024,         
        const uint32 segment_size = 128 * 1024 * 1024,  
        const uint32 occ_intv     = 128);               

    ~PagedText();

    uint32 page_count() const { return uint32( m_offsets.size() - 1u ); }

    uint64 size() const { return m_offsets.size() ? m_offsets.back() : 0u; }

    void grow();

    word_type* alloc_page();

    void release_page(word_type* page);

    const word_type* get_page(const uint32 i) const { return m_pages[i]; }

    uint32 get_page_size(const uint32 i) const { return uint32( m_offsets[i+1] - m_offsets[i] ); }

    uint64 get_page_offset(const uint32 i) const { return m_offsets[i]; }

    const uint32* get_occ_page(const uint32 i) const { return (uint32*)( m_pages[i] + m_page_size ); }

    uint32 find_page(const uint64 i) const;

    uint8 operator[] (const uint64 i) const;

    uint64 rank(const uint64 i, const uint8 c) const;

    uint64 rank(const uint32 page_idx, const uint64 i, const uint8 c) const;

    void reserve_pages(const uint32 n_pages);

    void reserve_free_pages(const uint32 n_pages);

    void reserve(const uint64 n);

    uint64 needed_host_memory(const uint64 n) const;

    uint64 needed_device_memory(const uint64 n) const { return 0u; }

    void resize(const uint64 n, const uint8* c = NULL);

    void insert(const uint32 n, const uint64* g, const uint8* c);

    const uint64* symbol_frequencies() const;

    uint64 symbol_frequency(const uint8 c) const { return symbol_frequencies()[c]; }

    void defrag();

    uint32                              m_page_size;        
    uint32                              m_segment_size;     
    uint32                              m_occ_intv;         
    uint32                              m_occ_intv_w;       
    uint32                              m_occ_intv_log;     
    uint32                              m_page_count;       
    nvbio::vector<host_tag,word_type*>  m_segments;
    nvbio::vector<host_tag,word_type*>  m_pages;
    nvbio::vector<host_tag,word_type*>  m_new_pages;
    nvbio::vector<host_tag,uint64>      m_offsets;
    nvbio::vector<host_tag,uint64>      m_new_offsets;
    nvbio::vector<host_tag,uint64>      m_counters;
    nvbio::vector<host_tag,uint64>      m_new_counters;
    nvbio::vector<host_tag,uint32>      m_buckets;
    std::vector<word_type*>             m_pool;
    uint32                              m_pool_size;
    omp_lock_t                          m_lock;
    uint32                              m_count_table[256];
};

struct SparseSymbolSet
{
    static const uint32 LOG_BUCKET_SIZE = 20u;
    static const uint32     BUCKET_SIZE = 1u << LOG_BUCKET_SIZE;

    SparseSymbolSet() : m_n( 0 ), m_n_special( 0 ) {}

    uint32 size() const { return m_n_special; }

    void reserve(const uint64 n, const uint32 n_special);

    void set(const uint64 range, const uint32 n_special, const uint32* p, const uint32* id);

    void insert(
        const uint64    range,
        const uint32    n_block,
        const uint64*   g,
        const uint32    n_special,
        const uint32*   p_special,
        const uint64*   g_special,
        const uint32*   id_special);

    void set_range(const uint64 n);

    uint32 rank(const uint64 i) const
    {
        if (i < m_n)
        {
            const uint64 ii = i+1;
            const uint32 b  = uint32( ii >> LOG_BUCKET_SIZE );
            const uint32 lo = m_buckets[b];
            const uint32 hi = m_buckets[b+1];
            return lower_bound_index( ii, &m_pos[lo], hi - lo ) + lo;
        }
        else if (i == uint64(-1))
            return 0;
        else // i >= m_n
            return m_n_special;
    }

    const uint64* pos() const { return raw_pointer( m_pos ); }
    const uint64* ids() const { return raw_pointer( m_id ); }

    uint64                              m_n;
    uint32                              m_n_special;
    nvbio::vector<host_tag,uint64>      m_pos;
    nvbio::vector<host_tag,uint64>      m_new_pos;
    nvbio::vector<host_tag,uint64>      m_id;
    nvbio::vector<host_tag,uint64>      m_new_id;
    nvbio::vector<host_tag,uint32>      m_buckets;
};

} // namespace nvbio

#include <nvbio/fmindex/paged_text_inl.h>