diff --git a/src/build/common.gypi b/src/build/common.gypi index 7cc506eebf..b85fe119a8 100644 --- a/src/build/common.gypi +++ b/src/build/common.gypi @@ -27,6 +27,10 @@ # TODO(ajm): we'd like to set this based on the target OS/architecture. 'prefer_fixed_point%': 0, + # Enable data logging. Produces text files with data logged within engines + # which can be easily parsed for offline processing. + 'enable_data_logging%': 0, + 'conditions': [ ['OS=="win"', { # TODO(ajm, perkj): does this need to be here? diff --git a/src/system_wrappers/interface/data_log.h b/src/system_wrappers/interface/data_log.h new file mode 100644 index 0000000000..4771f21ff2 --- /dev/null +++ b/src/system_wrappers/interface/data_log.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This singleton can be used for logging data for offline processing. Data + * logged with it can conveniently be parsed and processed with e.g. Matlab. + * + * Following is an example of the log file format, starting with the header + * row at line 1, and the data rows following. + * col1,col2,col3,multi-value-col4[3],,,col5 + * 123,10.2,-243,1,2,3,100 + * 241,12.3,233,1,2,3,200 + * 13,16.4,-13,1,2,3,300 + * + * As can be seen in the example, a multi-value-column is specified with the + * name followed the number of elements it contains. This followed by + * number of elements - 1 empty columns. + * + * Without multi-value-columns this format can be natively by Matlab. With + * multi-value-columns a small Matlab script is needed, available at + * trunk/tools/matlab/parseLog.m. + * + * Table names and column names are case sensitive. + */ + +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_H_ + +#include + +#include "data_log_impl.h" + +namespace webrtc { + +class DataLog { + public: + // Creates a log which uses a separate thread (referred to as the file + // writer thread) for writing log rows to file. + // + // Calls to this function after the log object has been created will only + // increment the reference counter. + static int CreateLog(); + + // Decrements the reference counter and deletes the log when the counter + // reaches 0. Should be called equal number of times as successful calls to + // CreateLog or memory leak will occur. + static void ReturnLog(); + + // Adds a new table, with the name table_name, and creates the file, with + // the name file_name, to which the table will be written. + // table_name is case sensitive. + static int AddTable(const std::string& table_name, + const std::string& file_name); + + // Adds a new column to a table. The column will be a multi-value-column + // if multi_value_length is greater than 1. + // Both table_name and column_name are case sensitive. + static int AddColumn(const std::string& table_name, + const std::string& column_name, + int multi_value_length); + + // Inserts a single value into a table with name table_name at the column + // with name column_name. + // Note that the ValueContainer makes use of the copy constructor, + // operator= and operator<< of the type T, and that the template type must + // implement a deep copy copy constructor and operator=. + // Copy constructor and operator= must not be disabled for the type T. + // Both table_name and column_name are case sensitive. + template + static int InsertCell(const std::string& table_name, + const std::string& column_name, + T value) { + DataLogImpl* data_log = DataLogImpl::StaticInstance(); + if (data_log == NULL) + return -1; + return data_log->InsertCell( + table_name, + column_name, + new ValueContainer(value)); + } + + // Inserts an array of values into a table with name table_name at the + // column specified by column_name, which must be a multi-value-column. + // Note that the MultiValueContainer makes use of the copy constructor, + // operator= and operator<< of the type T, and that the template type + // must implement a deep copy copy constructor and operator=. + // Copy constructor and operator= must not be disabled for the type T. + // Both table_name and column_name are case sensitive. + template + static int InsertCell(const std::string& table_name, + const std::string& column_name, + const T* array, + int length) { + DataLogImpl* data_log = DataLogImpl::StaticInstance(); + if (data_log == NULL) + return -1; + return data_log->InsertCell( + table_name, + column_name, + new MultiValueContainer(array, length)); + } + + // For the table table_name: Writes the current row to file. Starts a new + // empty row. + // table_name is case sensitive. + static int NextRow(const std::string& table_name); +}; + +} // namespace webrtc + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_H_ diff --git a/src/system_wrappers/interface/data_log_impl.h b/src/system_wrappers/interface/data_log_impl.h new file mode 100644 index 0000000000..67fc5486dd --- /dev/null +++ b/src/system_wrappers/interface/data_log_impl.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file contains the helper classes for the DataLog APIs. See data_log.h + * for the APIs. + * + * These classes are helper classes used for logging data for offline + * processing. Data logged with these classes can conveniently be parsed and + * processed with e.g. Matlab. + */ +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_IMPL_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_IMPL_H_ + +#include +#include +#include +#include + +#include "scoped_ptr.h" +#include "typedefs.h" + +namespace webrtc { + +class CriticalSectionWrapper; +class EventWrapper; +class LogTable; +class RWLockWrapper; +class ThreadWrapper; + +// All container classes need to implement a ToString-function to be +// writable to file. Enforce this via the Container interface. +class Container { + public: + virtual ~Container() {} + + virtual void ToString(std::string* container_string) const = 0; +}; + +template +class ValueContainer : public Container { + public: + explicit ValueContainer(T data) : data_(data) {} + + virtual void ToString(std::string* container_string) const { + *container_string = ""; + std::stringstream ss; + ss << data_ << ","; + ss >> *container_string; + } + + private: + T data_; +}; + +template +class MultiValueContainer : public Container { + public: + MultiValueContainer(const T* data, int length) + : data_(data, data + length) { + } + + virtual void ToString(std::string* container_string) const { + *container_string = ""; + std::stringstream ss; + for (size_t i = 0; i < data_.size(); ++i) + ss << data_[i] << ","; + *container_string += ss.str(); + } + + private: + std::vector data_; +}; + +class DataLogImpl { + public: + ~DataLogImpl(); + + // Creates a log which uses a separate thread (referred to as the file + // writer thread) for writing log rows to file. + // + // Calls to this function after the log object has been created will only + // increment the reference counter. + static int CreateLog(); + + // Returns a pointer to the instance of DataLogImpl which was created by + // CreateLog(), or NULL if no instance has been created, or if the + // previous instance has been deleted. + static DataLogImpl* StaticInstance(); + + // Decrements the reference counter keeping track of the number of times + // CreateLog() has been called. When the reference counter reaches 0 + // the DataLogImpl instance created by CreateLog() will be deleted. + // Should be called equal number of times as successful calls to + // CreateLog or memory leak will occur. + static void ReturnLog(); + + // Adds a new table, with the name table_name, and creates the file, with + // the name file_name, to which the table will be written. + // table_name is case sensitive. + int AddTable(const std::string& table_name, const std::string& file_name); + + // Adds a new column to a table. The column will be a multi-value-column + // if multi_value_length is greater than 1. + // Both table_name and column_name are case sensitive. + int AddColumn(const std::string& table_name, + const std::string& column_name, + int multi_value_length); + + // Inserts a Container into a table with name table_name at the column + // with name column_name. + // Both table_name and column_name are case sensitive. + int InsertCell(const std::string& table_name, + const std::string& column_name, + const Container* value_container); + + // For the table table_name: Writes the current row to file. Starts a new + // empty row. + // table_name is case sensitive. + int NextRow(const std::string& table_name); + + private: + DataLogImpl(); + + // Initializes the DataLogImpl object, allocates and starts the + // thread file_writer_thread_. + int Init(); + + // Write all complete rows in every table to file. + // This function should only be called by the file_writer_thread_ if that + // thread is running to avoid race conditions. + void Flush(); + + // Run() is called by the thread file_writer_thread_. + static bool Run(void* obj); + + // This function writes data to file. Note, it blocks if there is no data + // that should be written to file availble. Flush is the non-blocking + // version of this function. + void Process(); + + // Stops the continuous calling of Process(). + void StopThread(); + + // Collection of tables indexed by table name as std::string + typedef std::map TableMap; + typedef webrtc::scoped_ptr CritSectScopedPtr; + + static CritSectScopedPtr crit_sect_; + static DataLogImpl* instance_; + int counter_; + TableMap tables_; + EventWrapper* flush_event_; + ThreadWrapper* file_writer_thread_; + RWLockWrapper* tables_lock_; +}; + +} // namespace webrtc + +#endif // WEBRTC_SYSTEM_WRAPPERS_INTERFACE_DATA_LOG_IMPL_H_ diff --git a/src/system_wrappers/interface/scoped_ptr.h b/src/system_wrappers/interface/scoped_ptr.h new file mode 100644 index 0000000000..74b6ad365d --- /dev/null +++ b/src/system_wrappers/interface/scoped_ptr.h @@ -0,0 +1,258 @@ +// (C) Copyright Greg Colvin and Beman Dawes 1998, 1999. +// Copyright (c) 2001, 2002 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/smart_ptr/scoped_ptr.htm for documentation. +// + +// scoped_ptr mimics a built-in pointer except that it guarantees deletion +// of the object pointed to, either on destruction of the scoped_ptr or via +// an explicit reset(). scoped_ptr is a simple solution for simple needs; +// use shared_ptr or std::auto_ptr if your needs are more complex. + +// scoped_ptr_malloc added in by Google. When one of +// these goes out of scope, instead of doing a delete or delete[], it +// calls free(). scoped_ptr_malloc is likely to see much more +// use than any other specializations. + +// release() added in by Google. Use this to conditionally +// transfer ownership of a heap-allocated object to the caller, usually on +// method success. +#ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_SCOPED_PTR_H_ +#define WEBRTC_SYSTEM_WRAPPERS_INTERFACE_SCOPED_PTR_H_ + +#include // for assert +#include // for free() decl + +#include // for std::ptrdiff_t + +#ifdef _WIN32 +namespace std { using ::ptrdiff_t; }; +#endif // _WIN32 + +namespace webrtc { + +template +class scoped_ptr { + private: + + T* ptr; + + scoped_ptr(scoped_ptr const &); + scoped_ptr & operator=(scoped_ptr const &); + + public: + + typedef T element_type; + + explicit scoped_ptr(T* p = NULL): ptr(p) {} + + ~scoped_ptr() { + typedef char type_must_be_complete[sizeof(T)]; + delete ptr; + } + + void reset(T* p = NULL) { + typedef char type_must_be_complete[sizeof(T)]; + + if (ptr != p) { + T* obj = ptr; + ptr = p; + // Delete last, in case obj destructor indirectly results in ~scoped_ptr + delete obj; + } + } + + T& operator*() const { + assert(ptr != NULL); + return *ptr; + } + + T* operator->() const { + assert(ptr != NULL); + return ptr; + } + + T* get() const { + return ptr; + } + + void swap(scoped_ptr & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = NULL; + return tmp; + } + + T** accept() { + if (ptr) { + delete ptr; + ptr = NULL; + } + return &ptr; + } + + T** use() { + return &ptr; + } +}; + +template inline +void swap(scoped_ptr& a, scoped_ptr& b) { + a.swap(b); +} + + + + +// scoped_array extends scoped_ptr to arrays. Deletion of the array pointed to +// is guaranteed, either on destruction of the scoped_array or via an explicit +// reset(). Use shared_array or std::vector if your needs are more complex. + +template +class scoped_array { + private: + + T* ptr; + + scoped_array(scoped_array const &); + scoped_array & operator=(scoped_array const &); + + public: + + typedef T element_type; + + explicit scoped_array(T* p = NULL) : ptr(p) {} + + ~scoped_array() { + typedef char type_must_be_complete[sizeof(T)]; + delete[] ptr; + } + + void reset(T* p = NULL) { + typedef char type_must_be_complete[sizeof(T)]; + + if (ptr != p) { + T* arr = ptr; + ptr = p; + // Delete last, in case arr destructor indirectly results in ~scoped_array + delete [] arr; + } + } + + T& operator[](std::ptrdiff_t i) const { + assert(ptr != NULL); + assert(i >= 0); + return ptr[i]; + } + + T* get() const { + return ptr; + } + + void swap(scoped_array & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = NULL; + return tmp; + } + + T** accept() { + if (ptr) { + delete [] ptr; + ptr = NULL; + } + return &ptr; + } +}; + +template inline +void swap(scoped_array& a, scoped_array& b) { + a.swap(b); +} + +// scoped_ptr_malloc<> is similar to scoped_ptr<>, but it accepts a +// second template argument, the function used to free the object. + +template class scoped_ptr_malloc { + private: + + T* ptr; + + scoped_ptr_malloc(scoped_ptr_malloc const &); + scoped_ptr_malloc & operator=(scoped_ptr_malloc const &); + + public: + + typedef T element_type; + + explicit scoped_ptr_malloc(T* p = 0): ptr(p) {} + + ~scoped_ptr_malloc() { + FF(static_cast(ptr)); + } + + void reset(T* p = 0) { + if (ptr != p) { + FF(static_cast(ptr)); + ptr = p; + } + } + + T& operator*() const { + assert(ptr != 0); + return *ptr; + } + + T* operator->() const { + assert(ptr != 0); + return ptr; + } + + T* get() const { + return ptr; + } + + void swap(scoped_ptr_malloc & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + T** accept() { + if (ptr) { + FF(static_cast(ptr)); + ptr = 0; + } + return &ptr; + } +}; + +template inline +void swap(scoped_ptr_malloc& a, scoped_ptr_malloc& b) { + a.swap(b); +} + +} // namespace webrtc + +#endif // #ifndef WEBRTC_SYSTEM_WRAPPERS_INTERFACE_SCOPED_PTR_H_ diff --git a/src/system_wrappers/source/data_log.cc b/src/system_wrappers/source/data_log.cc new file mode 100644 index 0000000000..805f68c191 --- /dev/null +++ b/src/system_wrappers/source/data_log.cc @@ -0,0 +1,444 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "data_log.h" + +#include + +#include + +#include "critical_section_wrapper.h" +#include "event_wrapper.h" +#include "file_wrapper.h" +#include "rw_lock_wrapper.h" +#include "thread_wrapper.h" + +namespace webrtc { + +DataLogImpl::CritSectScopedPtr DataLogImpl::crit_sect_( + CriticalSectionWrapper::CreateCriticalSection()); + +DataLogImpl* DataLogImpl::instance_ = NULL; + +// A Row contains cells, which are indexed by the column names as std::string. +// The string index is case sensitive. +class Row { + public: + Row(); + ~Row(); + + // Inserts a Container into the cell of the column specified with + // column_name. + // column_name is case sensitive. + int InsertCell(const std::string& column_name, + const Container* value_container); + + // Converts the value at the column specified by column_name to a string + // stored in value_string. + // column_name is case sensitive. + void ToString(const std::string& column_name, std::string* value_string); + + private: + // Collection of containers indexed by column name as std::string + typedef std::map CellMap; + + CellMap cells_; + CriticalSectionWrapper* cells_lock_; +}; + +// A LogTable contains multiple rows, where only the latest row is active for +// editing. The rows are defined by the ColumnMap, which contains the name of +// each column and the length of the column (1 for one-value-columns and greater +// than 1 for multi-value-columns). +class LogTable { + public: + LogTable(); + ~LogTable(); + + // Adds the column with name column_name to the table. The column will be a + // multi-value-column if multi_value_length is greater than 1. + // column_name is case sensitive. + int AddColumn(const std::string& column_name, int multi_value_length); + + // Buffers the current row while it is waiting to be written to file, + // which is done by a call to Flush(). A new row is available when the + // function returns + void NextRow(); + + // Inserts a Container into the cell of the column specified with + // column_name. + // column_name is case sensitive. + int InsertCell(const std::string& column_name, + const Container* value_container); + + // Creates a log file, named as specified in the string file_name, to + // where the table will be written when calling Flush(). + int CreateLogFile(const std::string& file_name); + + // Write all complete rows to file. + // May not be called by two threads simultaneously (doing so may result in + // a race condition). Will be called by the file_writer_thread_ when that + // thread is running. + void Flush(); + + private: + // Collection of multi_value_lengths indexed by column name as std::string + typedef std::map ColumnMap; + typedef std::list RowList; + + ColumnMap columns_; + RowList rows_[2]; + RowList* rows_history_; + RowList* rows_flush_; + Row* current_row_; + FileWrapper* file_; + bool write_header_; + CriticalSectionWrapper* table_lock_; +}; + +Row::Row() + : cells_(), + cells_lock_(CriticalSectionWrapper::CreateCriticalSection()) { +} + +Row::~Row() { + for (CellMap::iterator it = cells_.begin(); it != cells_.end();) { + delete it->second; + // For maps all iterators (except the erased) are valid after an erase + cells_.erase(it++); + } + delete cells_lock_; +} + +int Row::InsertCell(const std::string& column_name, + const Container* value_container) { + CriticalSectionScoped synchronize(*cells_lock_); + assert(cells_.count(column_name) == 0); + if (cells_.count(column_name) > 0) + return -1; + cells_[column_name] = value_container; + return 0; +} + +void Row::ToString(const std::string& column_name, + std::string* value_string) { + CriticalSectionScoped synchronize(*cells_lock_); + const Container* container = cells_[column_name]; + if (container == NULL) { + *value_string = "NaN,"; + return; + } + container->ToString(value_string); +} + +LogTable::LogTable() + : columns_(), + rows_(), + rows_history_(&rows_[0]), + rows_flush_(&rows_[1]), + current_row_(new Row), + file_(FileWrapper::Create()), + write_header_(true), + table_lock_(CriticalSectionWrapper::CreateCriticalSection()) { +} + +LogTable::~LogTable() { + for (RowList::iterator row_it = rows_history_->begin(); + row_it != rows_history_->end();) { + delete *row_it; + row_it = rows_history_->erase(row_it); + } + for (ColumnMap::iterator col_it = columns_.begin(); + col_it != columns_.end();) { + // For maps all iterators (except the erased) are valid after an erase + columns_.erase(col_it++); + } + if (file_ != NULL) { + file_->Flush(); + file_->CloseFile(); + delete file_; + } + delete current_row_; + delete table_lock_; +} + +int LogTable::AddColumn(const std::string& column_name, + int multi_value_length) { + assert(multi_value_length > 0); + if (!write_header_) { + // It's not allowed to add new columns after the header + // has been written. + assert(false); + return -1; + } else { + CriticalSectionScoped synchronize(*table_lock_); + if (write_header_) + columns_[column_name] = multi_value_length; + else + return -1; + } + return 0; +} + +void LogTable::NextRow() { + CriticalSectionScoped sync_rows(*table_lock_); + rows_history_->push_back(current_row_); + current_row_ = new Row; +} + +int LogTable::InsertCell(const std::string& column_name, + const Container* value_container) { + CriticalSectionScoped synchronize(*table_lock_); + assert(columns_.count(column_name) > 0); + if (columns_.count(column_name) == 0) + return -1; + return current_row_->InsertCell(column_name, value_container); +} + +int LogTable::CreateLogFile(const std::string& file_name) { + if (file_name.length() == 0) + return -1; + if (file_->Open()) + return -1; + file_->OpenFile(file_name.c_str(), + false, // Open with read/write permissions + false, // Don't wraparound and write at the beginning when + // the file is full + true); // Open as a text file + if (file_ == NULL) + return -1; + return 0; +} + +void LogTable::Flush() { + ColumnMap::iterator column_it; + bool commit_header = false; + if (write_header_) { + CriticalSectionScoped synchronize(*table_lock_); + if (write_header_) { + commit_header = true; + write_header_ = false; + } + } + if (commit_header) { + for (column_it = columns_.begin(); + column_it != columns_.end(); ++column_it) { + if (column_it->second > 1) { + file_->WriteText("%s[%u],", column_it->first.c_str(), + column_it->second); + for (int i = 1; i < column_it->second; ++i) + file_->WriteText(","); + } else { + file_->WriteText("%s,", column_it->first.c_str()); + } + } + if (columns_.size() > 0) + file_->WriteText("\n"); + } + + // Swap the list used for flushing with the list containing the row history + // and clear the history. We also create a local pointer to the new + // list used for flushing to avoid race conditions if another thread + // calls this function while we are writing. + // We don't want to block the list while we're writing to file. + { + CriticalSectionScoped synchronize(*table_lock_); + RowList* tmp = rows_flush_; + rows_flush_ = rows_history_; + rows_history_ = tmp; + rows_history_->clear(); + } + + // Write all complete rows to file and delete them + for (RowList::iterator row_it = rows_flush_->begin(); + row_it != rows_flush_->end();) { + for (column_it = columns_.begin(); + column_it != columns_.end(); ++column_it) { + std::string row_string; + (*row_it)->ToString(column_it->first, &row_string); + file_->WriteText("%s", row_string.c_str()); + } + if (columns_.size() > 0) + file_->WriteText("\n"); + delete *row_it; + row_it = rows_flush_->erase(row_it); + } +} + +int DataLog::CreateLog() { + return DataLogImpl::CreateLog(); +} + +void DataLog::ReturnLog() { + return DataLogImpl::ReturnLog(); +} + +int DataLog::AddTable(const std::string& table_name, + const std::string& file_name) { + DataLogImpl* data_log = DataLogImpl::StaticInstance(); + if (data_log == NULL) + return -1; + return data_log->AddTable(table_name, file_name); +} + +int DataLog::AddColumn(const std::string& table_name, + const std::string& column_name, + int multi_value_length) { + DataLogImpl* data_log = DataLogImpl::StaticInstance(); + if (data_log == NULL) + return -1; + return data_log->DataLogImpl::StaticInstance()->AddColumn(table_name, + column_name, + multi_value_length); +} + +int DataLog::NextRow(const std::string& table_name) { + DataLogImpl* data_log = DataLogImpl::StaticInstance(); + if (data_log == NULL) + return -1; + return data_log->DataLogImpl::StaticInstance()->NextRow(table_name); +} + +DataLogImpl::DataLogImpl() + : counter_(1), + tables_(), + flush_event_(EventWrapper::Create()), + file_writer_thread_(NULL), + tables_lock_(RWLockWrapper::CreateRWLock()) { +} + +DataLogImpl::~DataLogImpl() { + StopThread(); + Flush(); // Write any remaining rows + delete file_writer_thread_; + delete flush_event_; + for (TableMap::iterator it = tables_.begin(); it != tables_.end();) { + delete static_cast(it->second); + // For maps all iterators (except the erased) are valid after an erase + tables_.erase(it++); + } + delete tables_lock_; +} + +int DataLogImpl::CreateLog() { + CriticalSectionScoped synchronize(*crit_sect_); + if (instance_ == NULL) { + instance_ = new DataLogImpl(); + return instance_->Init(); + } else { + ++instance_->counter_; + } + return 0; +} + +int DataLogImpl::Init() { + file_writer_thread_ = ThreadWrapper::CreateThread( + DataLogImpl::Run, + instance_, + kHighestPriority, + "DataLog"); + if (file_writer_thread_ == NULL) + return -1; + unsigned int thread_id = 0; + bool success = file_writer_thread_->Start(thread_id); + if (!success) + return -1; + return 0; +} + +DataLogImpl* DataLogImpl::StaticInstance() { + return instance_; +} + +void DataLogImpl::ReturnLog() { + CriticalSectionScoped synchronize(*crit_sect_); + if (instance_ && instance_->counter_ > 1) { + --instance_->counter_; + return; + } + delete instance_; + instance_ = NULL; +} + +int DataLogImpl::AddTable(const std::string& table_name, + const std::string& file_name) { + WriteLockScoped synchronize(*tables_lock_); + // Make sure we don't add a table which already exists + if (tables_.count(table_name) > 0) + return -1; + tables_[table_name] = new LogTable(); + if (tables_[table_name]->CreateLogFile(file_name) == -1) + return -1; + return 0; +} + +int DataLogImpl::AddColumn(const std::string& table_name, + const std::string& column_name, + int multi_value_length) { + ReadLockScoped synchronize(*tables_lock_); + if (tables_.count(table_name) == 0) + return -1; + return tables_[table_name]->AddColumn(column_name, multi_value_length); +} + +int DataLogImpl::InsertCell(const std::string& table_name, + const std::string& column_name, + const Container* value_container) { + ReadLockScoped synchronize(*tables_lock_); + assert(tables_.count(table_name) > 0); + if (tables_.count(table_name) == 0) + return -1; + return tables_[table_name]->InsertCell(column_name, value_container); +} + +int DataLogImpl::NextRow(const std::string& table_name) { + ReadLockScoped synchronize(*tables_lock_); + if (tables_.count(table_name) == 0) + return -1; + tables_[table_name]->NextRow(); + if (file_writer_thread_ == NULL) { + // Write every row to file as they get complete. + tables_[table_name]->Flush(); + } else { + // Signal a complete row + flush_event_->Set(); + } + return 0; +} + +void DataLogImpl::Flush() { + ReadLockScoped synchronize(*tables_lock_); + for (TableMap::iterator it = tables_.begin(); it != tables_.end(); ++it) { + it->second->Flush(); + } +} + +bool DataLogImpl::Run(void* obj) { + static_cast(obj)->Process(); + return true; +} + +void DataLogImpl::Process() { + // Wait for a row to be complete + flush_event_->Wait(WEBRTC_EVENT_INFINITE); + Flush(); +} + +void DataLogImpl::StopThread() { + if (file_writer_thread_ != NULL) { + file_writer_thread_->SetNotAlive(); + flush_event_->Set(); + // Call Stop() repeatedly, waiting for the Flush() call in Process() to + // finish. + while (!file_writer_thread_->Stop()) continue; + } +} + +} // namespace webrtc diff --git a/src/system_wrappers/source/data_log_dummy.cc b/src/system_wrappers/source/data_log_dummy.cc new file mode 100644 index 0000000000..21837aed96 --- /dev/null +++ b/src/system_wrappers/source/data_log_dummy.cc @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "data_log.h" + +namespace webrtc { + +int DataLog::CreateLog() { + return 0; +} + +void DataLog::ReturnLog() { +} + +int DataLog::AddTable(const std::string& /*table_name*/, + const std::string& /*file_name*/) { + return 0; +} + +int DataLog::AddColumn(const std::string& /*table_name*/, + const std::string& /*column_name*/, + int /*multi_value_length*/) { + return 0; +} + +int DataLog::NextRow(const std::string& /*table_name*/) { + return 0; +} + +DataLogImpl::DataLogImpl() { +} + +DataLogImpl::~DataLogImpl() { +} + +DataLogImpl* DataLogImpl::StaticInstance() { + return NULL; +} + +void DataLogImpl::ReturnLog() { +} + +int DataLogImpl::AddTable(const std::string& /*table_name*/, + const std::string& /*file_name*/) { + return 0; +} + +int DataLogImpl::AddColumn(const std::string& /*table_name*/, + const std::string& /*column_name*/, + int /*multi_value_length*/) { + return 0; +} + +int DataLogImpl::InsertCell(const std::string& /*table_name*/, + const std::string& /*column_name*/, + const Container* /*value_container*/) { + return 0; +} + +int DataLogImpl::NextRow(const std::string& /*table_name*/) { + return 0; +} + +void DataLogImpl::Flush() { +} + +bool DataLogImpl::Run(void* /*obj*/) { + return true; +} + +void DataLogImpl::Process() { +} + +void DataLogImpl::StopThread() { +} + +} // namespace webrtc diff --git a/src/system_wrappers/source/data_log_helpers_unittest.cc b/src/system_wrappers/source/data_log_helpers_unittest.cc new file mode 100644 index 0000000000..94b4d6ef59 --- /dev/null +++ b/src/system_wrappers/source/data_log_helpers_unittest.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "data_log.h" +#include "gtest/gtest.h" + +using ::webrtc::DataLog; + +TEST(TestDataLog, IntContainers) { + int c = 5; + webrtc::ValueContainer v1(c); + c = 10; + webrtc::ValueContainer v2(c); + std::string s1, s2; + v1.ToString(&s1); + v2.ToString(&s2); + ASSERT_EQ(s1, "5,"); + ASSERT_EQ(s2, "10,"); + v1 = v2; + v1.ToString(&s1); + ASSERT_EQ(s1, s2); +} + +TEST(TestDataLog, DoubleContainers) { + double c = 3.5; + webrtc::ValueContainer v1(c); + c = 10.3; + webrtc::ValueContainer v2(c); + std::string s1, s2; + v1.ToString(&s1); + v2.ToString(&s2); + ASSERT_EQ(s1, "3.5,"); + ASSERT_EQ(s2, "10.3,"); + v1 = v2; + v1.ToString(&s1); + ASSERT_EQ(s1, s2); +} + +TEST(TestDataLog, MultiValueContainers) { + int a[3] = {1, 2, 3}; + int b[3] = {4, 5, 6}; + webrtc::MultiValueContainer m1(a, 3); + webrtc::MultiValueContainer m2(b, 3); + webrtc::MultiValueContainer m3(a, 3); + std::string s1, s2, s3; + m1.ToString(&s1); + m2.ToString(&s2); + ASSERT_EQ(s1, "1,2,3,"); + ASSERT_EQ(s2, "4,5,6,"); + m1 = m2; + m1.ToString(&s1); + ASSERT_EQ(s1, s2); + m3.ToString(&s3); + ASSERT_EQ(s3, "1,2,3,"); +} diff --git a/src/system_wrappers/source/data_log_unittest.cc b/src/system_wrappers/source/data_log_unittest.cc new file mode 100644 index 0000000000..60318a77a9 --- /dev/null +++ b/src/system_wrappers/source/data_log_unittest.cc @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "data_log.h" +#include "gtest/gtest.h" + +using ::webrtc::DataLog; + +// A class for storing the values expected from a log table column when +// verifying a log table file. +struct ExpectedValues { + public: + ExpectedValues() + : values(NULL), + multi_value_length(1) { + } + + ExpectedValues(std::vector expected_values, + int expected_multi_value_length) + : values(expected_values), + multi_value_length(expected_multi_value_length) { + } + + std::vector values; + int multi_value_length; +}; + +typedef std::map ExpectedValuesMap; + +// A static class used for parsing and verifying data log files. +class DataLogParser { + public: + // Verifies that the log table stored in the file "log_file" corresponds to + // the cells and columns specified in "columns". + static int VerifyTable(FILE* log_file, const ExpectedValuesMap& columns) { + int row = 0; + char line_buffer[kMaxLineLength]; + char* ret = fgets(line_buffer, kMaxLineLength, log_file); + EXPECT_FALSE(ret == NULL); + if (ret == NULL) + return -1; + + std::string line(line_buffer, kMaxLineLength); + VerifyHeader(line, columns); + while (fgets(line_buffer, kMaxLineLength, log_file) != NULL) { + line = std::string(line_buffer, kMaxLineLength); + size_t line_position = 0; + + for (ExpectedValuesMap::const_iterator it = columns.begin(); + it != columns.end(); ++it) { + std::string str = ParseElement(line, &line_position, + it->second.multi_value_length); + EXPECT_EQ(str, it->second.values[row]); + if (str != it->second.values[row]) + return -1; + } + ++row; + } + return 0; + } + + // Verifies the table header stored in "line" to correspond with the header + // specified in "columns". + static int VerifyHeader(const std::string& line, + const ExpectedValuesMap& columns) { + size_t line_position = 0; + for (ExpectedValuesMap::const_iterator it = columns.begin(); + it != columns.end(); ++it) { + std::string str = ParseElement(line, &line_position, + it->second.multi_value_length); + EXPECT_EQ(str, it->first); + if (str != it->first) + return -1; + } + return 0; + } + + // Parses out and returns one element from the string "line", which contains + // one line read from a log table file. An element can either be a column + // header or a cell of a row. + static std::string ParseElement(const std::string& line, + size_t* line_position, + int multi_value_length) { + std::string parsed_cell; + parsed_cell = ""; + for (int i = 0; i < multi_value_length; ++i) { + size_t next_separator = line.find(',', *line_position); + EXPECT_NE(next_separator, std::string::npos); + if (next_separator == std::string::npos) + break; + parsed_cell += line.substr(*line_position, + next_separator - *line_position + 1); + *line_position = next_separator + 1; + } + return parsed_cell; + } + + // This constant defines the maximum line length the DataLogParser can + // parse. + enum { kMaxLineLength = 100 }; +}; + +TEST(TestDataLog, CreateReturnTest) { + for (int i = 0; i < 10; ++i) + ASSERT_EQ(DataLog::CreateLog(), 0); + ASSERT_EQ(DataLog::AddTable("a proper table", "table.txt"), 0); + for (int i = 0; i < 10; ++i) + DataLog::ReturnLog(); + ASSERT_LT(DataLog::AddTable("table failure", "table.txt"), 0); +} + +TEST(TestDataLog, VerifySingleTable) { + DataLog::CreateLog(); + DataLog::AddTable("table1", "table1.txt"); + DataLog::AddColumn("table1", "arrival", 1); + DataLog::AddColumn("table1", "timestamp", 1); + DataLog::AddColumn("table1", "size", 5); + WebRtc_UWord32 sizes[5] = {1400, 1500, 1600, 1700, 1800}; + for (int i = 0; i < 10; ++i) { + DataLog::InsertCell("table1", "arrival", static_cast(i)); + DataLog::InsertCell("table1", "timestamp", + static_cast(4354 + i)); + DataLog::InsertCell("table1", "size", sizes, 5); + DataLog::NextRow("table1"); + } + DataLog::ReturnLog(); + // Verify file + FILE* table = fopen("table1.txt", "r"); + ASSERT_FALSE(table == NULL); + // Read the column names and verify with the expected columns. + // Note that the columns are written to file in alphabetical order. + // Data expected from parsing the file + const int kNumberOfRows = 10; + std::string string_arrival[kNumberOfRows] = { + "0,", "1,", "2,", "3,", "4,", + "5,", "6,", "7,", "8,", "9," + }; + std::string string_timestamp[kNumberOfRows] = { + "4354,", "4355,", "4356,", "4357,", + "4358,", "4359,", "4360,", "4361,", + "4362,", "4363," + }; + std::string string_sizes = "1400,1500,1600,1700,1800,"; + ExpectedValuesMap expected; + expected["arrival,"] = ExpectedValues( + std::vector(string_arrival, + string_arrival + + kNumberOfRows), + 1); + expected["size[5],,,,,"] = ExpectedValues( + std::vector(10, string_sizes), 5); + expected["timestamp,"] = ExpectedValues( + std::vector(string_timestamp, + string_timestamp + + kNumberOfRows), + 1); + ASSERT_EQ(DataLogParser::VerifyTable(table, expected), 0); + fclose(table); +} + +TEST(TestDataLog, VerifyMultipleTables) { + DataLog::CreateLog(); + DataLog::AddTable("table2", "table2.txt"); + DataLog::AddTable("table3", "table3.txt"); + DataLog::AddColumn("table2", "arrival", 1); + DataLog::AddColumn("table2", "timestamp", 1); + DataLog::AddColumn("table2", "size", 1); + DataLog::AddTable("table4", "table4.txt"); + DataLog::AddColumn("table3", "timestamp", 1); + DataLog::AddColumn("table3", "arrival", 1); + DataLog::AddColumn("table4", "size", 1); + for (WebRtc_Word32 i = 0; i < 10; ++i) { + DataLog::InsertCell("table2", "arrival", + static_cast(i)); + DataLog::InsertCell("table2", "timestamp", + static_cast(4354 + i)); + DataLog::InsertCell("table2", "size", + static_cast(1200 + 10 * i)); + DataLog::InsertCell("table3", "timestamp", + static_cast(4354 + i)); + DataLog::InsertCell("table3", "arrival", + static_cast(i)); + DataLog::InsertCell("table4", "size", + static_cast(1200 + 10 * i)); + DataLog::NextRow("table4"); + DataLog::NextRow("table2"); + DataLog::NextRow("table3"); + } + DataLog::ReturnLog(); + + // Data expected from parsing the file + const int kNumberOfRows = 10; + std::string string_arrival[kNumberOfRows] = { + "0,", "1,", "2,", "3,", "4,", + "5,", "6,", "7,", "8,", "9," + }; + std::string string_timestamp[kNumberOfRows] = { + "4354,", "4355,", "4356,", "4357,", + "4358,", "4359,", "4360,", "4361,", + "4362,", "4363," + }; + std::string string_size[kNumberOfRows] = { + "1200,", "1210,", "1220,", "1230,", + "1240,", "1250,", "1260,", "1270,", + "1280,", "1290," + }; + + // Verify table 2 + { + FILE* table = fopen("table2.txt", "r"); + ASSERT_FALSE(table == NULL); + ExpectedValuesMap expected; + expected["arrival,"] = ExpectedValues( + std::vector(string_arrival, + string_arrival + + kNumberOfRows), + 1); + expected["size,"] = ExpectedValues( + std::vector(string_size, + string_size + kNumberOfRows), + 1); + expected["timestamp,"] = ExpectedValues( + std::vector(string_timestamp, + string_timestamp + + kNumberOfRows), + 1); + ASSERT_EQ(DataLogParser::VerifyTable(table, expected), 0); + fclose(table); + } + + // Verify table 3 + { + FILE* table = fopen("table3.txt", "r"); + ASSERT_FALSE(table == NULL); + ExpectedValuesMap expected; + expected["arrival,"] = ExpectedValues( + std::vector(string_arrival, + string_arrival + + sizeof(string_arrival) + / sizeof(std::string)), + 1); + expected["timestamp,"] = ExpectedValues( + std::vector(string_timestamp, + string_timestamp + + sizeof(string_timestamp) / + sizeof(std::string)), + 1); + ASSERT_EQ(DataLogParser::VerifyTable(table, expected), 0); + fclose(table); + } + + // Verify table 4 + { + FILE* table = fopen("table4.txt", "r"); + ASSERT_FALSE(table == NULL); + ExpectedValuesMap expected; + expected["size,"] = ExpectedValues( + std::vector(string_size, + string_size + + sizeof(string_size) + / sizeof(std::string)), + 1); + ASSERT_EQ(DataLogParser::VerifyTable(table, expected), 0); + fclose(table); + } +} diff --git a/src/system_wrappers/source/system_wrappers.gyp b/src/system_wrappers/source/system_wrappers.gyp index 728ef8c1ed..a16cd73c4c 100644 --- a/src/system_wrappers/source/system_wrappers.gyp +++ b/src/system_wrappers/source/system_wrappers.gyp @@ -28,11 +28,14 @@ '../interface/cpu_wrapper.h', '../interface/cpu_features_wrapper.h', '../interface/critical_section_wrapper.h', + '../interface/data_log.h', + '../interface/data_log_impl.h', '../interface/event_wrapper.h', '../interface/file_wrapper.h', '../interface/list_wrapper.h', '../interface/map_wrapper.h', '../interface/rw_lock_wrapper.h', + '../interface/scoped_ptr.h', '../interface/sort.h', '../interface/thread_wrapper.h', '../interface/tick_util.h', @@ -83,6 +86,15 @@ 'trace_posix.cc', ], }], + ['enable_data_logging==1', { + 'sources': [ + 'data_log.cc', + ], + },{ + 'sources': [ + 'data_log_dummy.cc', + ], + },], ['OS=="linux"', { 'sources': [ 'cpu_linux.cc', diff --git a/src/system_wrappers/source/system_wrappers_tests.gyp b/src/system_wrappers/source/system_wrappers_tests.gyp index 856f0c1033..5005d866f7 100644 --- a/src/system_wrappers/source/system_wrappers_tests.gyp +++ b/src/system_wrappers/source/system_wrappers_tests.gyp @@ -25,6 +25,14 @@ 'sources': [ 'list_unittest.cc', 'map_unittest.cc', + 'data_log_helpers_unittest.cc', + ], + 'conditions': [ + ['enable_data_logging==1', { + 'sources': [ + 'data_log_unittest.cc', + ], + },], ], }, ],