Add: translation normalization

Author: sysadm

README.md

@@ -1,7 +1,7 @@
 # webgate.pro
 
 [![CI](https://github.com/WebgateSystems/webgate.pro/actions/workflows/rubyonrails.yml/badge.svg)](https://github.com/WebgateSystems/webgate.pro/actions/workflows/rubyonrails.yml)
-![Coverage](https://img.shields.io/badge/coverage-95.1%25-brightgreen)
+![Coverage](https://img.shields.io/badge/coverage-95.2%25-brightgreen)
 [![Ruby](https://img.shields.io/badge/Ruby-3.2.2-CC342D?logo=ruby&logoColor=white)](https://www.ruby-lang.org/)
 [![Rails](https://img.shields.io/badge/Rails-7.0.10-D30001?logo=rubyonrails&logoColor=white)](https://rubyonrails.org/)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)

app/helpers/application_helper.rb

@@ -23,7 +23,9 @@ def compare_path(menu_item)
   end
 
   def main_menu_path(menu_item)
-    URI.escape(menu_item.name.mb_chars.downcase.to_s)
+    name = menu_item&.name
+    slug_source = name.presence || menu_item&.altlink.presence || ''
+    URI.escape(slug_source.to_s.mb_chars.downcase.to_s)
   end
 
   def menu_item_active?(menu_item)

app/services/gpt_translation_repair_service.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'openssl'
+require 'uri'
+
+class GptTranslationRepairService
+  class Error < StandardError; end
+
+  ENDPOINT = 'https://api.openai.com/v1/chat/completions'
+  MODEL = 'gpt-4o-mini'
+
+  def initialize(api_key: Settings.gpt_key)
+    @api_key = api_key
+  end
+
+  # Returns repaired HTML (string) in target language, with styling removed.
+  def call(base_html:, base_locale:, current_target_html:, target_locale:)
+    ensure_api_key!
+
+    target_locale = normalize_locale!(target_locale, label: 'target')
+    base_locale = normalize_locale!(base_locale, label: 'base')
+
+    base_html = base_html.to_s
+    current_target_html = current_target_html.to_s
+
+    prompt = build_prompt(base_html:, base_locale:, current_target_html:, target_locale:)
+    raw = chat(prompt)
+    parsed = parse_json(raw)
+    html = parsed.fetch('html')
+
+    HtmlTranslationNormalizer.call(html)
+  end
+
+  private
+
+  def ensure_api_key!
+    raise Error, 'Missing Settings.gpt_key' if @api_key.blank?
+  end
+
+  def normalize_locale!(locale, label:)
+    value = locale.to_s
+    raise Error, "Invalid #{label} locale: #{value}" if value.blank?
+
+    value
+  end
+
+  def build_prompt(base_html:, base_locale:, current_target_html:, target_locale:)
+    <<~PROMPT
+      You are a professional translator and HTML cleaner.
+
+      Task:
+      - Target language: #{target_locale.upcase}
+      - You will receive:
+        (A) Source HTML in #{base_locale.upcase} (source of truth)
+        (B) Current #{target_locale.upcase} HTML (may be wrong language, may contain inline styles/classes)
+      - Output MUST be a JSON object with exactly one key: "html".
+      - Value of "html" MUST be valid HTML and MUST contain ZERO inline styles and ZERO styling attributes:
+        remove all style="", class="", id="" attributes.
+      - Preserve structure and tags, but remove redundant wrapper spans if needed.
+      - If (B) is already in the correct target language, keep its meaning but normalize/clean the HTML.
+      - If (B) is NOT in the correct target language, translate from (A) to #{target_locale.upcase} and output cleaned HTML.
+      - Do not output markdown. Do not include explanations.
+
+      Source HTML (A):
+      #{base_html}
+
+      Current target HTML (B):
+      #{current_target_html}
+    PROMPT
+  end
+
+  def chat(prompt)
+    uri = URI.parse(ENDPOINT)
+    http = build_http(uri)
+    req = build_request(uri, prompt)
+    res = http.request(req)
+    parse_chat_response(res)
+  rescue JSON::ParserError => e
+    raise Error, "OpenAI JSON parse error: #{e.message}"
+  end
+
+  def build_http(uri)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    # Some servers fail SSL verification due to missing CRL/CA chain (e.g. "unable to get certificate CRL").
+    # For this internal task we intentionally disable verification.
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    http
+  end
+
+  def build_request(uri, prompt)
+    req = Net::HTTP::Post.new(uri.request_uri)
+    req['Authorization'] = "Bearer #{@api_key}"
+    req['Content-Type'] = 'application/json'
+    req.body = JSON.dump(build_payload(prompt))
+    req
+  end
+
+  def build_payload(prompt)
+    {
+      model: MODEL,
+      temperature: 0.2,
+      messages: [
+        { role: 'system', content: 'Return JSON only.' },
+        { role: 'user', content: prompt }
+      ]
+    }
+  end
+
+  def parse_chat_response(res)
+    raise Error, "OpenAI HTTP #{res.code}: #{res.body.to_s[0..300]}" unless res.is_a?(Net::HTTPSuccess)
+
+    body = JSON.parse(res.body)
+    content = body.dig('choices', 0, 'message', 'content')
+    raise Error, "OpenAI response missing content: #{res.body.to_s[0..300]}" if content.blank?
+
+    content
+  end
+
+  def parse_json(text)
+    JSON.parse(text)
+  rescue JSON::ParserError
+    # Try to extract JSON substring if model wrapped it
+    extracted = text.to_s[/\{[\s\S]*\}/]
+    raise Error, "Response is not valid JSON: #{text.to_s[0..300]}" if extracted.blank?
+
+    JSON.parse(extracted)
+  end
+end

app/services/html_translation_normalizer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class HtmlTranslationNormalizer
+  # Removes inline styling and “styling attributes” while keeping semantic HTML.
+  #
+  # - Strips: style, class, id
+  # - Unwraps <span> nodes that become attribute-less wrappers
+  #
+  # This is intentionally conservative: it does not remove tags like <strong>, <em>, <a>, etc.
+  def self.call(html)
+    return '' if html.blank?
+
+    s = html.to_s.dup
+
+    # Remove styling-related attributes (double or single quotes)
+    s.gsub!(/\s+(style|class|id)=(["']).*?\2/i, '')
+
+    # If spans became plain wrappers after stripping attributes, unwrap them.
+    # Also unwrap spans that were already attribute-less.
+    s.gsub!(/<span\s*>/i, '')
+    s.gsub!(%r{</span>}i, '')
+
+    s
+  end
+end

app/services/text_translation_normalizer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class TextTranslationNormalizer
+  # Cleans common artifacts like:
+  # - leading/trailing quotes (", ', “ ”, „)
+  # - trailing stray quotes after newlines
+  # - excessive whitespace/newlines
+  #
+  # It is conservative: it mainly trims and removes obvious wrapping/dangling quotes.
+  def self.call(text)
+    return '' if text.nil?
+
+    str = text.to_s
+    str = str.tr("\u00A0", ' ') # nbsp
+    str = str.strip
+    str = str.gsub(/\r\n?/, "\n")
+    str = str.gsub(/\n{3,}/, "\n\n")
+
+    str = remove_wrapping_quotes(str)
+    str = remove_dangling_trailing_quotes(str)
+    str.strip
+  end
+
+  QUOTE_PAIRS = [
+    ['"', '"'],
+    ["'", "'"],
+    ['“', '”'],
+    ['„', '”']
+  ].freeze
+
+  def self.remove_wrapping_quotes(str)
+    QUOTE_PAIRS.each do |open_q, close_q|
+      next unless str.start_with?(open_q) && str.end_with?(close_q) && str.length >= 2
+
+      str = str[1..-2].strip
+    end
+
+    str
+  end
+
+  def self.remove_dangling_trailing_quotes(str)
+    # Remove trailing quote chars if they appear as obvious artifacts at the very end
+    str.sub(/\s*["'“”„]+\s*\z/, '')
+  end
+
+  private_class_method :remove_wrapping_quotes, :remove_dangling_trailing_quotes
+end

lib/tasks/cleanup_technology_translations.rake

@@ -0,0 +1,66 @@
+# rubocop:disable Metrics/BlockLength
+namespace :technologies do
+  desc 'Clean technology translations (strip quotes, remove styles/classes/ids). ' \
+       'Usage: rake technologies:cleanup_translations[technology_id,locale,dry_run]'
+  task :cleanup_translations, %i[technology_id locale dry_run] => :environment do |_t, args|
+    technology_id = args[:technology_id].presence
+    target_locale = args[:locale].presence&.to_sym
+    dry_run = args[:dry_run].to_s == 'true'
+
+    locales =
+      if target_locale
+        [target_locale]
+      else
+        I18n.available_locales
+      end
+
+    scope = technology_id ? Technology.where(id: technology_id) : Technology.all
+    total = scope.count
+
+    puts 'Starting cleanup of technology translations...'
+    puts "Dry run: #{dry_run}"
+    puts "Locales: #{locales.join(', ')}"
+    puts ''
+
+    processed = 0
+    changed = 0
+    errors = 0
+
+    scope.find_each do |tech|
+      processed += 1
+      puts "[#{processed}/#{total}] Technology ID: #{tech.id} (#{tech.title})"
+
+      locales.each do |locale|
+        I18n.with_locale(locale) do
+          before_desc = tech.description.to_s
+          before_link = tech.link.to_s
+
+          after_desc = TextTranslationNormalizer.call(HtmlTranslationNormalizer.call(before_desc))
+          after_link = TextTranslationNormalizer.call(before_link)
+          after_link = before_link if after_link.blank?
+
+          next if after_desc == before_desc && after_link == before_link
+
+          if dry_run
+            changed += 1
+            puts "  - #{locale}: WOULD CLEAN"
+            next
+          end
+
+          t = tech.translations.find_or_initialize_by(locale: locale.to_s)
+          t.description = after_desc
+          t.link = after_link
+          t.save!
+          changed += 1
+          puts "  - #{locale}: cleaned"
+        end
+      rescue StandardError => e
+        errors += 1
+        puts "  - #{locale}: ERROR #{e.class}: #{e.message}"
+      end
+    end
+
+    puts "\nDone. Processed: #{processed}, changed: #{changed}, errors: #{errors}"
+  end
+end
+# rubocop:enable Metrics/BlockLength