Refactored smartctl output parser for better API, support for different formats.

src/applib/CMakeLists.txt

@@ -30,6 +30,10 @@ target_sources(applib PRIVATE
 	gui_utils.h
 	selftest.cpp
 	selftest.h
+	smartctl_parser.cpp
+	smartctl_parser.h
+	smartctl_ata_json_parser.cpp
+	smartctl_ata_json_parser.h
 	smartctl_ata_text_parser.cpp
 	smartctl_ata_text_parser.h
 	smartctl_executor.cpp

src/applib/ata_storage_property_descr.cpp

@@ -19,6 +19,7 @@ Copyright:
 #include "applib/app_pcrecpp.h"
 
 #include "ata_storage_property_descr.h"
+#include "warning_colors.h"
 
 
 
@@ -1977,5 +1978,16 @@ WarningLevel ata_storage_property_autoset_warning(AtaStorageProperty& p)
 
 
 
+std::vector<AtaStorageProperty> StoragePropertyProcessor::process_properties(std::vector<AtaStorageProperty> properties, AtaStorageAttribute::DiskType disk_type)
+{
+	for (auto& p : properties) {
+		ata_storage_property_autoset_description(p, disk_type);
+		ata_storage_property_autoset_warning(p);
+		storage_property_autoset_warning_descr(p);  // append warning to description
+	}
+	return properties;
+}
+
+
 
 /// @}

src/applib/ata_storage_property_descr.h

@@ -16,12 +16,14 @@ Copyright:
 
 
 
-/// Fill the property with all the information we can gather (description, etc...).
-bool ata_storage_property_autoset_description(AtaStorageProperty& p, AtaStorageAttribute::DiskType disk_type);
+class StoragePropertyProcessor {
+	public:
 
+		/// Set descriptions, warnings, etc... on properties, and return them.
+		static std::vector<AtaStorageProperty> process_properties(std::vector<AtaStorageProperty> properties, AtaStorageAttribute::DiskType disk_type);
+
+};
 
-/// Do some basic checks on the property and set warnings if needed.
-WarningLevel ata_storage_property_autoset_warning(AtaStorageProperty& p);
 
 
 

src/applib/examples/example_smartctl_parser.cpp

@@ -42,13 +42,13 @@ int main(int argc, char* argv[])
 		return EXIT_FAILURE;
 	}
 
-	SmartctlAtaTextParser sp;
-	if (!sp.parse_full(contents, AtaStorageAttribute::DiskType::Any)) {
-		debug_out_error("app", "Cannot parse file contents: " << sp.get_error_msg() << "\n");
+	SmartctlAtaTextParser parser;
+	if (!parser.parse_full(contents)) {
+		debug_out_error("app", "Cannot parse file contents: " << parser.get_error_msg() << "\n");
 		return EXIT_FAILURE;
 	}
 
-	const std::vector<AtaStorageProperty>& props = sp.get_properties();
+	const std::vector<AtaStorageProperty>& props = parser.get_properties();
 	for(const auto& prop : props) {
 		debug_out_dump("app", prop << "\n");
 	}

src/applib/selftest.cpp

@@ -18,6 +18,7 @@ Copyright:
 #include "ata_storage_property.h"
 #include "smartctl_ata_text_parser.h"
 #include "selftest.h"
+#include "ata_storage_property_descr.h"
 
 
 
@@ -242,16 +243,19 @@ std::string SelfTest::update(const std::shared_ptr<CommandExecutor>& smartctl_ex
 		return error_msg;
 
 	AtaStorageAttribute::DiskType disk_type = drive_->get_is_hdd() ? AtaStorageAttribute::DiskType::Hdd : AtaStorageAttribute::DiskType::Ssd;
-	SmartctlAtaTextParser ps;
-	if (!ps.parse_full(output, disk_type)) {  // try to parse it
-		return ps.get_error_msg();
+	auto parser = SmartctlParser::create(SmartctlOutputParserType::Text);
+	DBG_ASSERT_RETURN(parser, "Cannot create parser");
+
+	if (!parser->parse_full(output)) {  // try to parse it
+		return parser->get_error_msg();
 	}
+	auto properties = StoragePropertyProcessor::process_properties(parser->get_properties(), disk_type);
 
 	// Note: Since the self-test log is sometimes late
 	// and in undetermined order (sorting by hours is too rough),
 	// we use the "self-test status" capability.
 	AtaStorageProperty p;
-	for (const auto& e : ps.get_properties()) {
+	for (const auto& e : properties) {
 // 		if (e.section != AtaStorageProperty::Section::data || e.subsection != AtaStorageProperty::SubSection::selftest_log
 		if (e.section != AtaStorageProperty::Section::internal
 				|| !e.is_value_type<AtaStorageSelftestEntry>() || e.get_value<AtaStorageSelftestEntry>().test_num != 0

src/applib/smartctl_ata_json_parser.cpp

@@ -0,0 +1,82 @@
+/******************************************************************************
+License: GNU General Public License v3.0 only
+Copyright:
+	(C) 2022 Alexander Shaduri <ashaduri@gmail.com>
+******************************************************************************/
+/// \file
+/// \author Alexander Shaduri
+/// \ingroup applib
+/// \weakgroup applib
+/// @{
+
+#include "smartctl_ata_json_parser.h"
+
+/*
+Information not printed in JSON yet:
+
+- Checksum warnings (smartctl.cpp: checksumwarning()).
+	Smartctl output: Warning! SMART <section name> Structure error: invalid SMART checksum
+ 	Keys:
+ 		_text_only/attribute_data_checksum_error
+ 		_text_only/attribute_thresholds_checksum_error
+ 		_text_only/ata_error_log_checksum_error
+ 		_text_only/selftest_log_checksum_error
+
+- Samsung warning
+ 	Smartctl output: May need -F samsung or -F samsung2 enabled; see manual for details
+ 	We ignore this in text parser.
+
+- Warnings from drivedb.h in the middle of Info section
+	Smartctl output (example):
+		WARNING: A firmware update for this drive may be available,
+		see the following Seagate web pages:
+		...
+	Keys: _text_only/info_warning
+
+- Errors about consistency:
+	"Invalid Error Log index ..."
+	"Warning: ATA error count %d inconsistent with error log pointer"
+	We ignore this in text parser.
+
+- "mandatory SMART command failed" and similar errors.
+	We ignore this in text parser.
+
+- SMART support and some other Info keys
+	_text_only/smart_supported
+	_text_only/smart_enabled
+	_text_only/write_cache_reorder
+	_text_only/power_mode
+
+- Automatic Offline Data Collection toggle support
+	text_only/aodc_support
+
+ - Directory log supported
+ 	We don't use this.
+ 	_text_only/directory_log_supported
+
+ata_smart_error_log/_not_present
+
+
+Keys:
+smartctl/version/_merged
+ 	Looks like "7.2"
+smartctl/version/_merged_full
+	Looks like "smartctl 7.2 2020-12-30 r5155", formed from "/smartctl" subkeys.
+
+_custom/smart_enabled
+ 	Not present in json?
+*/
+
+
+
+bool SmartctlAtaJsonParser::parse_full(const std::string& full)
+{
+	// TODO
+	return false;
+}
+
+
+
+
+
+/// @}

src/applib/smartctl_ata_json_parser.h

@@ -0,0 +1,42 @@
+/******************************************************************************
+License: GNU General Public License v3.0 only
+Copyright:
+	(C) 2022 Alexander Shaduri <ashaduri@gmail.com>
+******************************************************************************/
+/// \file
+/// \author Alexander Shaduri
+/// \ingroup applib
+/// \weakgroup applib
+/// @{
+
+#ifndef SMARTCTL_ATA_JSON_PARSER_H
+#define SMARTCTL_ATA_JSON_PARSER_H
+
+#include <string>
+#include <vector>
+
+#include "smartctl_parser.h"
+
+
+
+/// Smartctl (S)ATA text output parser.
+/// Note: ALL parse_* functions (except parse_full() and parse_version())
+/// expect data in unix-newline format!
+class SmartctlAtaJsonParser : public SmartctlParser {
+	public:
+
+		// Defaulted, used by make_unique.
+		SmartctlAtaJsonParser() = default;
+
+		// Overridden
+		bool parse_full(const std::string& full) override;
+
+
+};
+
+
+
+
+#endif
+
+/// @}

src/applib/smartctl_ata_text_parser.cpp

@@ -9,11 +9,11 @@ Copyright:
 /// \weakgroup applib
 /// @{
 
-#include "local_glibmm.h"
+// #include "local_glibmm.h"
 #include <clocale>  // localeconv
 #include <cstdint>
 
-#include "hz/locale_tools.h"  // ScopedCLocale, locale_c_get().
+// #include "hz/locale_tools.h"  // ScopedCLocale, locale_c_get().
 #include "hz/string_algo.h"  // string_*
 #include "hz/string_num.h"  // string_is_numeric, number_to_string
 #include "hz/debug.h"  // debug_*
@@ -21,7 +21,7 @@ Copyright:
 #include "app_pcrecpp.h"
 #include "smartctl_ata_text_parser.h"
 #include "ata_storage_property_descr.h"
-#include "warning_colors.h"
+// #include "warning_colors.h"
 #include "smartctl_version_parser.h"
 #include "smartctl_text_parser_helper.h"
 
@@ -68,14 +68,10 @@ namespace {
 
 
 // Parse full "smartctl -x" output
-bool SmartctlAtaTextParser::parse_full(const std::string& full, AtaStorageAttribute::DiskType disk_type)
+bool SmartctlAtaTextParser::parse_full(const std::string& full)
 {
-	this->clear();  // clear previous data
-
 	this->set_data_full(full);
 
-	disk_type_ = disk_type;
-
 
 	// -------------------- Fix the output so it doesn't interfere with proper parsing
 
@@ -2298,59 +2294,6 @@ ID      Size     Value  Description
 
 
 
-void SmartctlAtaTextParser::clear()
-{
-	data_full_.clear();
-	data_section_info_.clear();
-	data_section_data_.clear();
-	error_msg_.clear();
-
-	properties_.clear();
-}
-
-
-
-std::string SmartctlAtaTextParser::get_data_full() const
-{
-	return data_full_;
-}
-
-
-
-std::string SmartctlAtaTextParser::get_error_msg() const
-{
-	return Glib::ustring::compose(_("Cannot parse smartctl output: %1"), error_msg_);
-}
-
-
-
-const std::vector<AtaStorageProperty>& SmartctlAtaTextParser::get_properties() const
-{
-	return properties_;
-}
-
-
-
-// adds a property into property list, looks up and sets its description.
-// Yes, there's no place for this in the Parser, but whatever...
-void SmartctlAtaTextParser::add_property(AtaStorageProperty p)
-{
-	ata_storage_property_autoset_description(p, disk_type_);
-	ata_storage_property_autoset_warning(p);
-	storage_property_autoset_warning_descr(p);  // append warning to description
-
-	properties_.push_back(p);
-}
-
-
-
-void SmartctlAtaTextParser::set_data_full(const std::string& s)
-{
-	data_full_ = s;
-}
-
-
-
 void SmartctlAtaTextParser::set_data_section_info(const std::string& s)
 {
 	data_section_info_ = s;
@@ -2365,13 +2308,6 @@ void SmartctlAtaTextParser::set_data_section_data(const std::string& s)
 
 
 
-void SmartctlAtaTextParser::set_error_msg(const std::string& s)
-{
-	error_msg_ = s;
-}
-
-
-
 
 
 

src/applib/smartctl_ata_text_parser.h

@@ -1,7 +1,7 @@
 /******************************************************************************
 License: GNU General Public License v3.0 only
 Copyright:
-	(C) 2008 - 2021 Alexander Shaduri <ashaduri@gmail.com>
+	(C) 2008 - 2022 Alexander Shaduri <ashaduri@gmail.com>
 ******************************************************************************/
 /// \file
 /// \author Alexander Shaduri
@@ -15,21 +15,24 @@ Copyright:
 #include <string>
 #include <vector>
 
-#include "ata_storage_property.h"
+#include "smartctl_parser.h"
 
 
 
 /// Smartctl (S)ATA text output parser.
 /// Note: ALL parse_* functions (except parse_full() and parse_version())
 /// expect data in unix-newline format!
-class SmartctlAtaTextParser {
+class SmartctlAtaTextParser : public SmartctlParser {
 	public:
 
-		/// Parse full "smartctl -x" output
-		bool parse_full(const std::string& full, AtaStorageAttribute::DiskType disk_type);
+		// Defaulted, used by make_unique.
+		SmartctlAtaTextParser() = default;
+
+		// Overridden
+		bool parse_full(const std::string& full) override;
 
 
-	private:
+	protected:
 
 		/// Parse the section part (with "=== .... ===" header) - info or data sections.
 		bool parse_section(const std::string& header, const std::string& body);
@@ -63,59 +66,18 @@ class SmartctlAtaTextParser {
 		bool parse_section_data_internal_capabilities(AtaStorageProperty& cap_prop);
 
 
-		/// Clear parsed data
-		void clear();
-
-
-	public:
-
-		/// Get "full" data, as passed to parse_full().
-		[[nodiscard]] std::string get_data_full() const;
-
-		/// Get parse error message. Call this only if parsing doesn't succeed,
-		/// to get a friendly error message.
-		[[nodiscard]] std::string get_error_msg() const;
-
-
-		/// Get parse result properties
-		[[nodiscard]] const std::vector<AtaStorageProperty>& get_properties() const;
-
-
-
-	private:
-
-
-		/// Add a property into property list, look up and set its description
-		void add_property(AtaStorageProperty p);
-
-
-		/// Set "full" data ("smartctl -x" output)
-		void set_data_full(const std::string& s);
-
-
 		/// Set "info" section data ("smartctl -i" output, or the first part of "smartctl -x" output)
 		void set_data_section_info(const std::string& s);
 
-
 		/// Parse "data" section data (the second part of "smartctl -x" output).
 		void set_data_section_data(const std::string& s);
 
 
-		/// Set error message
-		void set_error_msg(const std::string& s);
+	private:
 
-
-
-		std::vector<AtaStorageProperty> properties_;  ///< Parsed data properties
-
-		std::string data_full_;  ///< full data, filled by parse_full()
 		std::string data_section_info_;  ///< "info" section data, filled by parse_section_info()
 		std::string data_section_data_;  ///< "data" section data, filled by parse_section_data()
 
-		std::string error_msg_;  ///< This will be filled with some displayable message on error
-
-		AtaStorageAttribute::DiskType disk_type_ = AtaStorageAttribute::DiskType::Any;  ///< Disk type (HDD, SSD)
-
 };
 
 

src/applib/smartctl_parser.cpp

@@ -0,0 +1,101 @@
+/******************************************************************************
+License: GNU General Public License v3.0 only
+Copyright:
+	(C) 2022 Alexander Shaduri <ashaduri@gmail.com>
+******************************************************************************/
+/// \file
+/// \author Alexander Shaduri
+/// \ingroup applib
+/// \weakgroup applib
+/// @{
+
+#include <locale>
+#include <cctype>  // isspace
+
+#include "smartctl_parser.h"
+#include "smartctl_ata_text_parser.h"
+#include "smartctl_ata_json_parser.h"
+#include "ata_storage_property_descr.h"
+#include "warning_colors.h"
+
+
+
+std::unique_ptr<SmartctlParser> SmartctlParser::create(SmartctlOutputParserType type)
+{
+	switch(type) {
+		case SmartctlOutputParserType::Auto:
+			break;
+		case SmartctlOutputParserType::Json:
+			return std::make_unique<SmartctlAtaJsonParser>();
+		case SmartctlOutputParserType::Text:
+			return std::make_unique<SmartctlAtaTextParser>();
+	}
+	return nullptr;
+}
+
+
+
+std::optional<SmartctlOutputParserType> SmartctlParser::detect_output_type(const std::string& output) const
+{
+	// Look for the first non-whitespace symbol
+	auto first_symbol = std::find_if(output.begin(), output.end(), [&](char c) {
+		return !std::isspace(c, std::locale::classic());
+	});
+	if (first_symbol != output.end() && *first_symbol == '-'
+
+
+
+
+}
+
+
+
+std::string SmartctlParser::get_data_full() const
+{
+	return data_full_;
+}
+
+
+
+std::string SmartctlParser::get_error_msg() const
+{
+	return Glib::ustring::compose(_("Cannot parse smartctl output: %1"), error_msg_);
+}
+
+
+
+const std::vector<AtaStorageProperty>& SmartctlParser::get_properties() const
+{
+	return properties_;
+}
+
+
+
+// adds a property into property list, looks up and sets its description.
+// Yes, there's no place for this in the Parser, but whatever...
+void SmartctlParser::add_property(AtaStorageProperty p)
+{
+	properties_.push_back(std::move(p));
+}
+
+
+
+void SmartctlParser::set_data_full(const std::string& s)
+{
+	data_full_ = s;
+}
+
+
+
+void SmartctlParser::set_error_msg(const std::string& s)
+{
+	error_msg_ = s;
+}
+
+
+
+
+
+
+
+/// @}

src/applib/smartctl_parser.h

@@ -0,0 +1,108 @@
+/******************************************************************************
+License: GNU General Public License v3.0 only
+Copyright:
+	(C) 2022 Alexander Shaduri <ashaduri@gmail.com>
+******************************************************************************/
+/// \file
+/// \author Alexander Shaduri
+/// \ingroup applib
+/// \weakgroup applib
+/// @{
+
+#ifndef SMARTCTL_PARSER_H
+#define SMARTCTL_PARSER_H
+
+#include <string>
+#include <vector>
+#include <memory>
+
+#include "ata_storage_property.h"
+#include "smartctl_output_type.h"
+
+
+
+/// Smartctl (S)ATA text output parser.
+/// Note: ALL parse_* functions (except parse_full() and parse_version())
+/// expect data in unix-newline format!
+class SmartctlParser {
+	protected:
+
+		// Defaulted but hidden
+		SmartctlParser() = default;
+
+
+	public:
+
+		// Deleted
+		SmartctlParser(const SmartctlParser& other) = default;
+
+		// Deleted
+		SmartctlParser(SmartctlParser&& other) = delete;
+
+		// Deleted
+		SmartctlParser& operator=(const SmartctlParser& other) = delete;
+
+		// Deleted
+		SmartctlParser& operator=(SmartctlParser&& other) = delete;
+
+		/// Virtual member requirement
+		virtual ~SmartctlParser() = default;
+
+
+		/// Create an instance of this class.
+		/// \return nullptr if no such class exists
+		static std::unique_ptr<SmartctlParser> create(SmartctlOutputParserType type);
+
+
+		/// Create an instance of this class.
+		/// \return nullptr if no such class exists
+		static std::unique_ptr<SmartctlParser> detect_and_parse(SmartctlOutputParserType type);
+
+
+		/// Parse full "smartctl -x" output.
+		/// Note: Once parsed, this function cannot be called again.
+		virtual bool parse_full(const std::string& full) = 0;
+
+
+		/// Detect smartctl output type (text, json).
+		/// Return
+		[[nodiscard]] std::optional<SmartctlOutputParserType> detect_output_type(const std::string& output) const;
+
+
+		/// Get "full" data, as passed to parse_full().
+		[[nodiscard]] std::string get_data_full() const;
+
+		/// Get parse error message. Call this only if parsing doesn't succeed,
+		/// to get a friendly error message.
+		[[nodiscard]] std::string get_error_msg() const;
+
+		/// Get parse result properties
+		[[nodiscard]] const std::vector<AtaStorageProperty>& get_properties() const;
+
+
+	protected:
+
+		/// Add a property into property list, look up and set its description
+		void add_property(AtaStorageProperty p);
+
+		/// Set "full" data ("smartctl -x" output), json or text.
+		void set_data_full(const std::string& s);
+
+		/// Set error message
+		void set_error_msg(const std::string& s);
+
+
+	private:
+
+		std::vector<AtaStorageProperty> properties_;  ///< Parsed data properties
+		std::string data_full_;  ///< full data, filled by parse_full()
+		std::string error_msg_;  ///< This will be filled with some displayable message on error
+
+};
+
+
+
+
+#endif
+
+/// @}

src/applib/storage_device.cpp

@@ -24,6 +24,7 @@ Copyright:
 #include "smartctl_executor.h"
 #include "smartctl_version_parser.h"
 #include "smartctl_text_parser_helper.h"
+#include "ata_storage_property_descr.h"
 
 
 
@@ -153,7 +154,9 @@ std::string StorageDevice::parse_basic_data(bool do_set_properties, bool emit_si
 	// SMART support is: Unavailable - Packet Interface Devices [this device: CD/DVD] don't support ATA SMART
 	// Sample output line 2 (encountered on a BDRW drive):
 	// Device type:          CD/DVD
-	if (app_pcre_match("/this device: CD\\/DVD/mi", info_output_) || app_pcre_match("/^Device type:\\s+CD\\/DVD/mi", info_output_)) {
+	// NOTE: CD/DVD detection does not work in "-d scsi" mode.
+	if (app_pcre_match("/this device: CD\\/DVD/mi", info_output_)
+			|| app_pcre_match("/^Device type:\\s+CD\\/DVD/mi", info_output_)) {
 		debug_out_dump("app", "Drive " << get_device_with_type() << " seems to be a CD/DVD device.\n");
 		this->set_detected_type(DetectedType::cddvd);
 
@@ -237,9 +240,12 @@ std::string StorageDevice::parse_basic_data(bool do_set_properties, bool emit_si
 		if (hdd_.has_value()) {
 			disk_type = hdd_.value() ? AtaStorageAttribute::DiskType::Hdd : AtaStorageAttribute::DiskType::Ssd;
 		}
-		SmartctlAtaTextParser ps;
-		if (ps.parse_full(this->info_output_, disk_type)) {  // try to parse it
-			this->set_properties(ps.get_properties());  // copy to our drive, overwriting old data
+
+		auto parser = SmartctlParser::create(SmartctlOutputParserType::Text);
+		DBG_ASSERT_RETURN(parser, "Cannot create parser");
+
+		if (parser->parse_full(this->info_output_)) {  // try to parse it
+			this->set_properties(StoragePropertyProcessor::process_properties(parser->get_properties(), disk_type));  // copy to our drive, overwriting old data
 		}
 	}
 
@@ -301,11 +307,14 @@ std::string StorageDevice::parse_data()
 	if (hdd_.has_value()) {
 		disk_type = hdd_.value() ? AtaStorageAttribute::DiskType::Hdd : AtaStorageAttribute::DiskType::Ssd;
 	}
-	SmartctlAtaTextParser ps;
-	if (ps.parse_full(this->full_output_, disk_type)) {  // try to parse it (parse only, set the properties after basic parsing).
+
+	auto parser = SmartctlParser::create(SmartctlOutputParserType::Text);
+	DBG_ASSERT_RETURN(parser, "Cannot create parser");
+
+	if (parser->parse_full(this->full_output_)) {  // try to parse it (parse only, set the properties after basic parsing).
 
 		// refresh basic info too
-		this->info_output_ = ps.get_data_full();  // put data including version information
+		this->info_output_ = parser->get_data_full();  // put data including version information
 
 		// note: this will clear the non-basic properties!
 		// this will parse some info that is already parsed by SmartctlAtaTextParser::parse_full(),
@@ -315,8 +324,9 @@ std::string StorageDevice::parse_data()
 		// Call this after parse_basic_data(), since it sets parse status to "info".
 		this->set_parse_status(StorageDevice::ParseStatus::full);
 
-		// set the full properties
-		this->set_properties(ps.get_properties());  // copy to our drive, overwriting old data
+		// set the full properties.
+		// copy to our drive, overwriting old data.
+		this->set_properties(StoragePropertyProcessor::process_properties(parser->get_properties(), disk_type));
 
 		signal_changed().emit(this);  // notify listeners
 
@@ -331,7 +341,7 @@ std::string StorageDevice::parse_data()
 	// proper parsing failed. try to at least extract info section
 	this->info_output_ = this->full_output_;  // complete output here. sometimes it's only the info section
 	if (!this->parse_basic_data(true).empty()) {  // will add some properties too. this will emit signal_changed().
-		return ps.get_error_msg();  // return full parser's error messages - they are more detailed.
+		return parser->get_error_msg();  // return full parser's error messages - they are more detailed.
 	}
 
 	return {};  // return ok if at least the info was ok.