I am trying to decompress files with lzma using easylzma library,some files work good but random files does not decompress.
after some debugging i found that totalread number is greater by 1 than header uncompressedSize ,also the header isstreamed is 0 .
the code says that there is no footer but when i subtract 1 from total read to skip the error ,the file is decompressed correctly but with addition of one row at the end of file which has several fields as 0 and single field with value.
the files are .bi5 from dukascopy.
I want to determine if the error is due to some bad logic in the library I use,or it is the file which is bad and in this case what should be done.
the used libraries are easylzma-master and dukascopy-master from github and files are downloaded from dukascopy servers.
exactly the file of 13h_ticks.bi5 and 21_ticks.bi5 on 30 september 2020 "september is 8 " shows this problem.
update:
I did not put code because I am asking about guideline now,the code is present and it shows the problem.but it is library code.so i am wondering if some one had same problem regarding this specefic files of dukascopy bi5 type and this lzma library.I am just looking now for general rule of "when does in lzma decompression we get behaviour of having totalread is greater than header uncompressed size by 1 repeatedly??does this mean there is footer but it is not mentioned in header bytes??"
update:
this is how I open the file
int HTTPRequest::read_bi5_main(boost::filesystem::path p, ptime epoch)
{
boost::unique_lock<boost::mutex> read_bi5_to_bin_lock(mBOOST_LOGMutex,boost::defer_lock);
boost::unique_lock<boost::mutex> read_bi5_to_bin_lock2(m_read_bi5_to_binMutex, boost::defer_lock);
unsigned char *buffer;
size_t buffer_size;
int counter;
size_t raw_size = 0;
std::string filename_string = p.generic_string();
path p2 = p;
p2.replace_extension(".bin");
std::string filename_string_to_bin =p2.generic_string() ;
path p3 = p;
p3.replace_extension(".csv");
std::string filename_string_to_csv = p3.generic_string();
const char *filename = filename_string.c_str();
const char *filename_to_bin = filename_string_to_bin.c_str();
const char *filename_to_csv = filename_string_to_csv.c_str();
//22-9-2020 here I open the downloaded file if possible
if (fs::exists(p) && fs::is_regular(p))
{
buffer_size = fs::file_size(p);
buffer = new unsigned char[buffer_size];
}
else {
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Error: couldn't access the data file. |"
<< filename << "|" << std::endl;
read_bi5_to_bin_lock.unlock();
return 2;
}
//22-9-2020 here I read the downloaded file into filestream
std::ifstream fin(filename, std::ifstream::binary);
fin.read(reinterpret_cast<char*>(buffer), buffer_size);
fin.close();
//22-9-2020 here I check if file is related to japanese yen so that I determine how to write its value
/*
if symbols_xxx has mHTTPRequest_Symbol_str then PV=0.001
else if symbols_xxxx has mHTTPRequest_Symbol_str then PV=0.0001
else if symbols_xxxx has mHTTPRequest_Symbol_str then PV=0.00001
*/
//28-9-2020 I will make 3 vectors in utils.h for 3,4,5 point value ,then I find symbol in vector,
//std::size_t pos = mHTTPRequest_Symbol_str.find("JPY");
double PV;
std::vector<std::string>::iterator it3 = std::find(point_value_xxx.begin(), point_value_xxx.end(), mHTTPRequest_Symbol_str);
std::vector<std::string>::iterator it4 = std::find(point_value_xxxx.begin(), point_value_xxxx.end(), mHTTPRequest_Symbol_str);
std::vector<std::string>::iterator it5 = std::find(point_value_xxxxx.begin(), point_value_xxxxx.end(), mHTTPRequest_Symbol_str);
if (it3 != point_value_xxx.end())
{
PV = 0.001;
}
else if (it4 != point_value_xxxx.end())
{
PV = 0.0001;
}
else if (it5 != point_value_xxxxx.end())
{
PV = 0.00001;
}
else
{
//10-1-2020throw;
PV = 0.001;
}
read_bi5_to_bin_lock2.lock();
unsigned char *data_bin_buffer = 0 ;
n47::tick_data *data = n47::read_bi5_to_bin(
buffer, buffer_size, epoch, PV, &raw_size, &data_bin_buffer);
//5-11-2020 here i will save binary file
std::string file_name_path_string=output_compressed_file_2(&data_bin_buffer, raw_size, filename_to_bin);
read_bi5_to_bin_lock2.unlock();
path file_name_path_2{ file_name_path_string };
buffer_size = 0;
if (fs::exists(file_name_path_2) && fs::is_regular(file_name_path_2))
{
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << boost::this_thread::get_id() <<"\t we can access the data .bin file. |"
<< filename_to_bin << "| with size ="<< fs::file_size(file_name_path_2) << std::endl;
read_bi5_to_bin_lock.unlock();
}
else {
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Error: couldn't access the data .bin file. |"
<< filename_to_bin << "|" << std::endl;
read_bi5_to_bin_lock.unlock();
return 2;
}
n47::tick_data_iterator iter;
//5-11-2020 here i will save file.csv from data which is pointer to vector to pointers to ticks
if (data == 0)
{
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Failure: Failed to load the data!" << std::endl;
read_bi5_to_bin_lock.unlock();
}
//5-15-2020 take care that without else ,error happens with empty files because data is pointer to vector of pointers to ticks .so when data is made inside read_bi5 ,it is made as null pointer and later it is assigned to vector if file has ticks.if file does not have ticks ,then it is just returned as null pointer .so when dereferencing null pointer we got error
else if (data->size() != (raw_size / n47::ROW_SIZE))
{
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "Failure: Loaded " << data->size()
<< " ticks but file size indicates we should have loaded "
<< (raw_size / n47::ROW_SIZE) << std::endl;
read_bi5_to_bin_lock.unlock();
}
//22-9-2020 in last if and if else I checked if file is either empty or has error of data size So now I have good clean file to work with
//read_bi5_to_bin_lock.lock();
//BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << "time, bid, bid_vol, ask, ask_vol" << std::endl;
//read_bi5_to_bin_lock.unlock();
counter = 0;
std::ofstream out_csv(filename_string_to_csv);
if (data == 0)
{
}
else if (data != 0)
{
for (iter = data->begin(); iter != data->end(); iter++) {
//5-11-2020 here i will save file.csv from data which is pointer to vector to pointers to ticks>>>>>>>here i should open file stream for output and save data to it
out_csv
//<< std::setfill('0')<<std::setw(sizeof((*iter)->epoch + (*iter)->td))<<std::fixed<<((*iter)->epoch + (*iter)->td) << ","
//<< std::setfill('0')<<std::setw(27)<<std::fixed<<((*iter)->epoch + (*iter)->td) << ","
<< std::setfill('0')<<((*iter)->epoch + (*iter)->td) << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->bid)<<std::fixed << (*iter)->bid << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->bidv)<<std::fixed << (*iter)->bidv << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->ask)<<std::fixed << (*iter)->ask << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->askv)<<std::fixed << (*iter)->askv << std::endl;
//??5-17-2020 isolate multithreaded error
/*
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) <<
boost::this_thread::get_id() << "\t"<<((*iter)->epoch + (*iter)->td) << ", "
<< (*iter)->bid << ", " << (*iter)->bidv << ", "
<< (*iter)->ask << ", " << (*iter)->askv << std::endl;
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) <<
boost::this_thread::get_id() << "\t"<< std::setfill('0')<< std::setw(sizeof((*iter)->epoch + (*iter)->td))<<((*iter)->epoch + (*iter)->td) << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->bid)<< (*iter)->bid << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->bidv)<< (*iter)->bidv << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->ask)<< (*iter)->ask << ","
<< std::setfill('0')<<std::setw(sizeof(*iter)->askv)<< (*iter)->askv << std::endl;
read_bi5_to_bin_lock.unlock();
*/
counter++;
}
////read_bi5_to_bin_lock.unlock();
}
out_csv.close();
//5-13-2020
//??5-17-2020 isolate multithreaded error
read_bi5_to_bin_lock.lock();
BOOST_LOG((*mHTTPRequest_LoggingInstance_shared_pointer).mloggerCoutLog) << ".end." << std::endl << std::endl
<< "From " << raw_size << " bytes we read " << counter
<< " records." << std::endl
<< raw_size << " / " << n47::ROW_SIZE << " = "
<< (raw_size / n47::ROW_SIZE) << std::endl;
read_bi5_to_bin_lock.unlock();
delete data;
delete[] buffer;
delete [] data_bin_buffer;
return 0;
}
and this is my dukascopy modified file
//#include "stdafx.h"
/*
Copyright 2013 Michael O'Keeffe (a.k.a. ninety47).
This file is part of ninety47 Dukascopy toolbox.
The "ninety47 Dukascopy toolbox" is free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the License,
or any later version.
"ninety47 Dukascopy toolbox" is distributed in the hope that it will be
useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
Public License for more details.
You should have received a copy of the GNU General Public License along with
"ninety47 Dukascopy toolbox". If not, see <http://www.gnu.org/licenses/>.
*/
#include "ninety47/dukascopy.h"
#include <boost/date_time/posix_time/posix_time.hpp>
#include <algorithm>
#include <vector>
#include "ninety47/dukascopy/defs.h"
#include "ninety47/dukascopy/io.hpp"
#include "ninety47/dukascopy/lzma.h"
namespace n47 {
namespace pt = boost::posix_time;
tick *tickFromBuffer(
unsigned char *buffer, pt::ptime epoch, float digits, size_t offset) {
bytesTo<unsigned int, n47::BigEndian> bytesTo_unsigned;
bytesTo<float, n47::BigEndian> bytesTo_float;
unsigned int ts = bytesTo_unsigned(buffer + offset);
pt::time_duration ms = pt::millisec(ts);
unsigned int ofs = offset + sizeof(ts);
float ask = bytesTo_unsigned(buffer + ofs) * digits;
ofs += sizeof(ts);
float bid = bytesTo_unsigned(buffer + ofs) * digits;
ofs += sizeof(ts);
//28-9-2020 convert volume to million
float askv = bytesTo_float(buffer + ofs) *1000000;
ofs += sizeof(ts);
float bidv = bytesTo_float(buffer + ofs) *1000000;
return new tick(epoch, ms, ask, bid, askv, bidv);
}
tick_data* read_bin(
unsigned char *buffer, size_t buffer_size, pt::ptime epoch, float point_value) {
std::vector<tick*> *data = new std::vector<tick*>();
std::vector<tick*>::iterator iter;
std::size_t offset = 0;
while ( offset < buffer_size ) {
data->push_back(tickFromBuffer(buffer, epoch, point_value, offset));
offset += ROW_SIZE;
}
return data;
}
tick_data* read_bi5(
unsigned char *lzma_buffer, size_t lzma_buffer_size, pt::ptime epoch,
float point_value, size_t *bytes_read) {
tick_data *result = 0;
// decompress
int status;
unsigned char *buffer = n47::lzma::decompress(lzma_buffer,
lzma_buffer_size, &status, bytes_read);
//5-11-2020 here i will save binary file
if (status != N47_E_OK) {
bytes_read = 0;
} else {
// convert to tick data (with read_bin).
result = read_bin(buffer, *bytes_read, epoch, point_value);
delete [] buffer;
}
return result;
}
//5-11-2020
tick_data* read_bi5_to_bin(
unsigned char *lzma_buffer, size_t lzma_buffer_size, pt::ptime epoch,
float point_value, size_t *bytes_read, unsigned char** buffer_decompressed) {
tick_data *result = 0;
// decompress
int status;
*buffer_decompressed = n47::lzma::decompress(lzma_buffer,
lzma_buffer_size, &status, bytes_read);
if (status != N47_E_OK)
{
bytes_read = 0;
}
else {
// convert to tick data (with read_bin).
result = read_bin(*buffer_decompressed, *bytes_read, epoch, point_value);
//delete[] buffer;
}
return result;
}
tick_data* read(
const char *filename, pt::ptime epoch, float point_value, size_t *bytes_read) {
tick_data *result = 0;
size_t buffer_size = 0;
unsigned char *buffer = n47::io::loadToBuffer<unsigned char>(filename, &buffer_size);
if ( buffer != 0 ) {
if ( n47::lzma::bufferIsLZMA(buffer, buffer_size) ) {
result = read_bi5(buffer, buffer_size, epoch, point_value, bytes_read);
// Reading in as bi5 failed lets double check its not binary
// data in the buffer.
if (result == 0) {
result = read_bin(buffer, buffer_size, epoch, point_value);
}
} else {
result = read_bin(buffer, buffer_size, epoch, point_value);
*bytes_read = buffer_size;
}
delete [] buffer;
if (result != 0 && result->size() != (*bytes_read / n47::ROW_SIZE)) {
delete result;
result = 0;
}
}
return result;
}
} // namespace n47
I digged down till the details of lzma working,which is heavy for me, so I changed the library and used 7z cpp lzma spec file.
it works.
i think the problem is related to something about using c code in cpp program.
also the library states that it is tested for bsd thanks for your help. any one with same case download 7zip and use cpp files