Merge pull request #2819 from bunnei/telemetry-submit

Telemetry: Submit logged data to the Citra service

.gitmodules

@@ -28,3 +28,9 @@
 [submodule "externals/enet"]
     path = externals/enet
     url = https://github.com/lsalzman/enet
+[submodule "cpr"]
+    path = externals/cpr
+    url = https://github.com/whoshuu/cpr.git
+[submodule "json"]
+    path = externals/json
+    url = https://github.com/nlohmann/json.git

CMakeLists.txt

@@ -11,6 +11,8 @@ option(CITRA_USE_BUNDLED_SDL2 "Download bundled SDL2 binaries" OFF)
 option(ENABLE_QT "Enable the Qt frontend" ON)
 option(CITRA_USE_BUNDLED_QT "Download bundled Qt binaries" OFF)
 
+option(ENABLE_WEB_SERVICE "Enable web services (telemetry, etc.)" ON)
+
 if(NOT EXISTS ${CMAKE_SOURCE_DIR}/.git/hooks/pre-commit)
     message(STATUS "Copying pre-commit hook")
     file(COPY hooks/pre-commit
@@ -223,6 +225,9 @@ if (ENABLE_QT)
     find_package(Qt5 REQUIRED COMPONENTS Widgets OpenGL ${QT_PREFIX_HINT})
 endif()
 
+if (ENABLE_WEB_SERVICE)
+    add_definitions(-DENABLE_WEB_SERVICE)
+endif()
 
 # Platform-specific library requirements
 # ======================================

externals/CMakeLists.txt

@@ -52,3 +52,15 @@ endif()
 # ENet
 add_subdirectory(enet)
 target_include_directories(enet INTERFACE ./enet/include)
+
+if (ENABLE_WEB_SERVICE)
+    # CPR
+    option(BUILD_TESTING OFF)
+    option(BUILD_CPR_TESTS OFF)
+    add_subdirectory(cpr)
+    target_include_directories(cpr INTERFACE ./cpr/include)
+
+    # JSON
+    add_library(json-headers INTERFACE)
+    target_include_directories(json-headers INTERFACE ./json/src)
+endif()

externals/cpr

@@ -0,0 +1 @@
+Subproject commit b5758fbc88021437f968fe5174f121b8b92f5d5c

externals/json

@@ -0,0 +1 @@
+Subproject commit d3496347fcd1382896fca3aaf78a0d803c2f52ec

src/CMakeLists.txt

@@ -14,3 +14,6 @@ endif()
 if (ENABLE_QT)
     add_subdirectory(citra_qt)
 endif()
+if (ENABLE_WEB_SERVICE)
+    add_subdirectory(web_service)
+endif()

src/citra/config.cpp

@@ -151,6 +151,10 @@ void Config::ReadValues() {
     Settings::values.use_gdbstub = sdl2_config->GetBoolean("Debugging", "use_gdbstub", false);
     Settings::values.gdbstub_port =
         static_cast<u16>(sdl2_config->GetInteger("Debugging", "gdbstub_port", 24689));
+
+    // Web Service
+    Settings::values.telemetry_endpoint_url = sdl2_config->Get(
+        "WebService", "telemetry_endpoint_url", "https://services.citra-emu.org/api/telemetry");
 }
 
 void Config::Reload() {

src/citra/default_ini.h

@@ -168,5 +168,9 @@ log_filter = *:Info
 # Port for listening to GDB connections.
 use_gdbstub=false
 gdbstub_port=24689
+
+[WebService]
+# Endpoint URL for submitting telemetry data
+telemetry_endpoint_url =
 )";
 }

src/citra_qt/configuration/config.cpp

@@ -133,6 +133,13 @@ void Config::ReadValues() {
     Settings::values.gdbstub_port = qt_config->value("gdbstub_port", 24689).toInt();
     qt_config->endGroup();
 
+    qt_config->beginGroup("WebService");
+    Settings::values.telemetry_endpoint_url =
+        qt_config->value("telemetry_endpoint_url", "https://services.citra-emu.org/api/telemetry")
+            .toString()
+            .toStdString();
+    qt_config->endGroup();
+
     qt_config->beginGroup("UI");
 
     qt_config->beginGroup("UILayout");
@@ -268,6 +275,11 @@ void Config::SaveValues() {
     qt_config->setValue("gdbstub_port", Settings::values.gdbstub_port);
     qt_config->endGroup();
 
+    qt_config->beginGroup("WebService");
+    qt_config->setValue("telemetry_endpoint_url",
+                        QString::fromStdString(Settings::values.telemetry_endpoint_url));
+    qt_config->endGroup();
+
     qt_config->beginGroup("UI");
 
     qt_config->beginGroup("UILayout");

src/common/logging/backend.cpp

@@ -73,7 +73,8 @@ namespace Log {
     SUB(Audio, Sink)                                                                               \
     CLS(Input)                                                                                     \
     CLS(Network)                                                                                   \
-    CLS(Loader)
+    CLS(Loader)                                                                                    \
+    CLS(WebService)
 
 // GetClassName is a macro defined by Windows.h, grrr...
 const char* GetLogClassName(Class log_class) {

src/common/logging/log.h

@@ -91,6 +91,7 @@ enum class Class : ClassType {
     Loader,            ///< ROM loader
     Input,             ///< Input emulation
     Network,           ///< Network emulation
+    WebService,        ///< Interface to Citra Web Services
     Count              ///< Total number of logging classes
 };
 

src/core/CMakeLists.txt

@@ -388,3 +388,6 @@ create_directory_groups(${SRCS} ${HEADERS})
 add_library(core STATIC ${SRCS} ${HEADERS})
 target_link_libraries(core PUBLIC common PRIVATE audio_core video_core)
 target_link_libraries(core PUBLIC Boost::boost PRIVATE cryptopp dynarmic fmt)
+if (ENABLE_WEB_SERVICE)
+    target_link_libraries(core PUBLIC json-headers web_service)
+endif()

src/core/settings.h

@@ -126,6 +126,9 @@ struct Values {
     // Debugging
     bool use_gdbstub;
     u16 gdbstub_port;
+
+    // WebService
+    std::string telemetry_endpoint_url;
 } extern values;
 
 // a special value for Values::region_value indicating that citra will automatically select a region

src/core/telemetry_session.cpp

@@ -7,12 +7,18 @@
 #include "common/scm_rev.h"
 #include "core/telemetry_session.h"
 
+#ifdef ENABLE_WEB_SERVICE
+#include "web_service/telemetry_json.h"
+#endif
+
 namespace Core {
 
 TelemetrySession::TelemetrySession() {
-    // TODO(bunnei): Replace with a backend that logs to our web service
+#ifdef ENABLE_WEB_SERVICE
+    backend = std::make_unique<WebService::TelemetryJson>();
+#else
     backend = std::make_unique<Telemetry::NullVisitor>();
-
+#endif
     // Log one-time session start information
     const auto duration{std::chrono::steady_clock::now().time_since_epoch()};
     const auto start_time{std::chrono::duration_cast<std::chrono::microseconds>(duration).count()};

src/web_service/CMakeLists.txt

@@ -0,0 +1,14 @@
+set(SRCS
+            telemetry_json.cpp
+            web_backend.cpp
+            )
+
+set(HEADERS
+            telemetry_json.h
+            web_backend.h
+            )
+
+create_directory_groups(${SRCS} ${HEADERS})
+
+add_library(web_service STATIC ${SRCS} ${HEADERS})
+target_link_libraries(web_service PUBLIC common cpr json-headers)

src/web_service/telemetry_json.cpp

@@ -0,0 +1,87 @@
+// Copyright 2017 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "common/assert.h"
+#include "core/settings.h"
+#include "web_service/telemetry_json.h"
+#include "web_service/web_backend.h"
+
+namespace WebService {
+
+template <class T>
+void TelemetryJson::Serialize(Telemetry::FieldType type, const std::string& name, T value) {
+    sections[static_cast<u8>(type)][name] = value;
+}
+
+void TelemetryJson::SerializeSection(Telemetry::FieldType type, const std::string& name) {
+    TopSection()[name] = sections[static_cast<unsigned>(type)];
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<bool>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<double>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<float>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<u8>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<u16>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<u32>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<u64>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<s8>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<s16>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<s32>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<s64>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<std::string>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue());
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<const char*>& field) {
+    Serialize(field.GetType(), field.GetName(), std::string(field.GetValue()));
+}
+
+void TelemetryJson::Visit(const Telemetry::Field<std::chrono::microseconds>& field) {
+    Serialize(field.GetType(), field.GetName(), field.GetValue().count());
+}
+
+void TelemetryJson::Complete() {
+    SerializeSection(Telemetry::FieldType::App, "App");
+    SerializeSection(Telemetry::FieldType::Session, "Session");
+    SerializeSection(Telemetry::FieldType::Performance, "Performance");
+    SerializeSection(Telemetry::FieldType::UserFeedback, "UserFeedback");
+    SerializeSection(Telemetry::FieldType::UserConfig, "UserConfig");
+    SerializeSection(Telemetry::FieldType::UserSystem, "UserSystem");
+    PostJson(Settings::values.telemetry_endpoint_url, TopSection().dump());
+}
+
+} // namespace WebService

src/web_service/telemetry_json.h

@@ -0,0 +1,54 @@
+// Copyright 2017 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <array>
+#include <string>
+#include <json.hpp>
+#include "common/telemetry.h"
+
+namespace WebService {
+
+/**
+ * Implementation of VisitorInterface that serialized telemetry into JSON, and submits it to the
+ * Citra web service
+ */
+class TelemetryJson : public Telemetry::VisitorInterface {
+public:
+    TelemetryJson() = default;
+    ~TelemetryJson() = default;
+
+    void Visit(const Telemetry::Field<bool>& field) override;
+    void Visit(const Telemetry::Field<double>& field) override;
+    void Visit(const Telemetry::Field<float>& field) override;
+    void Visit(const Telemetry::Field<u8>& field) override;
+    void Visit(const Telemetry::Field<u16>& field) override;
+    void Visit(const Telemetry::Field<u32>& field) override;
+    void Visit(const Telemetry::Field<u64>& field) override;
+    void Visit(const Telemetry::Field<s8>& field) override;
+    void Visit(const Telemetry::Field<s16>& field) override;
+    void Visit(const Telemetry::Field<s32>& field) override;
+    void Visit(const Telemetry::Field<s64>& field) override;
+    void Visit(const Telemetry::Field<std::string>& field) override;
+    void Visit(const Telemetry::Field<const char*>& field) override;
+    void Visit(const Telemetry::Field<std::chrono::microseconds>& field) override;
+
+    void Complete() override;
+
+private:
+    nlohmann::json& TopSection() {
+        return sections[static_cast<u8>(Telemetry::FieldType::None)];
+    }
+
+    template <class T>
+    void Serialize(Telemetry::FieldType type, const std::string& name, T value);
+
+    void SerializeSection(Telemetry::FieldType type, const std::string& name);
+
+    nlohmann::json output;
+    std::array<nlohmann::json, 7> sections;
+};
+
+} // namespace WebService

src/web_service/web_backend.cpp

@@ -0,0 +1,52 @@
+// Copyright 2017 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <cpr/cpr.h>
+#include <stdlib.h>
+#include "common/logging/log.h"
+#include "web_service/web_backend.h"
+
+namespace WebService {
+
+static constexpr char API_VERSION[]{"1"};
+static constexpr char ENV_VAR_USERNAME[]{"CITRA_WEB_SERVICES_USERNAME"};
+static constexpr char ENV_VAR_TOKEN[]{"CITRA_WEB_SERVICES_TOKEN"};
+
+static std::string GetEnvironmentVariable(const char* name) {
+    const char* value{getenv(name)};
+    if (value) {
+        return value;
+    }
+    return {};
+}
+
+const std::string& GetUsername() {
+    static const std::string username{GetEnvironmentVariable(ENV_VAR_USERNAME)};
+    return username;
+}
+
+const std::string& GetToken() {
+    static const std::string token{GetEnvironmentVariable(ENV_VAR_TOKEN)};
+    return token;
+}
+
+void PostJson(const std::string& url, const std::string& data) {
+    if (url.empty()) {
+        LOG_ERROR(WebService, "URL is invalid");
+        return;
+    }
+
+    if (GetUsername().empty() || GetToken().empty()) {
+        LOG_ERROR(WebService, "Environment variables %s and %s must be set to POST JSON",
+                  ENV_VAR_USERNAME, ENV_VAR_TOKEN);
+        return;
+    }
+
+    cpr::PostAsync(cpr::Url{url}, cpr::Body{data}, cpr::Header{{"Content-Type", "application/json"},
+                                                               {"x-username", GetUsername()},
+                                                               {"x-token", GetToken()},
+                                                               {"api-version", API_VERSION}});
+}
+
+} // namespace WebService

src/web_service/web_backend.h

@@ -0,0 +1,31 @@
+// Copyright 2017 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <string>
+#include "common/common_types.h"
+
+namespace WebService {
+
+/**
+ * Gets the current username for accessing services.citra-emu.org.
+ * @returns Username as a string, empty if not set.
+ */
+const std::string& GetUsername();
+
+/**
+ * Gets the current token for accessing services.citra-emu.org.
+ * @returns Token as a string, empty if not set.
+ */
+const std::string& GetToken();
+
+/**
+ * Posts JSON to services.citra-emu.org.
+ * @param url URL of the services.citra-emu.org endpoint to post data to.
+ * @param data String of JSON data to use for the body of the POST request.
+ */
+void PostJson(const std::string& url, const std::string& data);
+
+} // namespace WebService