Bazel progress only proxy

Examples/BazelProgressXCBBuildService/BUILD.bazel

@@ -0,0 +1,27 @@
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library", "swift_proto_library")
+load("@build_bazel_rules_apple//apple:macos.bzl", "macos_command_line_application")
+
+swift_proto_library(
+    name = "build_event_stream_proto",
+    deps = ["@build_bazel_bazel//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_proto"],
+)
+
+swift_library(
+    name = "BazelProgressXCBBuildService.library",
+    module_name = "BazelProgressXCBBuildService",
+    srcs = glob(["Sources/**/*.swift"]),
+    deps = [
+        ":build_event_stream_proto",
+        "@com_github_apple_swift_log//:Logging",
+        "@com_github_apple_swift_nio//:NIO",
+        "@com_github_target_xcbbuildserviceproxy//:XCBBuildServiceProxy",
+        "@com_github_target_xcbbuildserviceproxy//:XCBProtocol",
+        "@com_github_target_xcbbuildserviceproxy//:XCBProtocol_13_0",
+    ],
+)
+
+macos_command_line_application(
+    name = "BazelProgressXCBBuildService",
+    minimum_os_version = "10.15",
+    deps = [":BazelProgressXCBBuildService.library"],
+)

Examples/BazelProgressXCBBuildService/Package.swift

@@ -0,0 +1,34 @@
+// swift-tools-version:5.1
+
+import PackageDescription
+
+let package = Package(
+    name: "BazelProgressXCBBuildService",
+    platforms: [.macOS(.v10_14)],
+    products: [
+        .executable(name: "BazelProgressXCBBuildService", targets: ["BazelProgressXCBBuildService"]),
+    ],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-log.git", from: "1.4.2"),
+        .package(url: "https://github.com/apple/swift-nio.git", from: "2.33.0"),
+        .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.9.0"),
+        // XCBBuildServiceProxy lives up two levels from here
+        .package(path: "../../"),
+        .package(path: "../BazelXCBBuildService/"),
+    ],
+    targets: [
+        .target(
+            name: "BazelProgressXCBBuildService",
+            dependencies: [
+                "src_main_java_com_google_devtools_build_lib_buildeventstream_proto_build_event_stream_proto",
+                "Logging",
+                "NIO",
+                "SwiftProtobuf",
+                "XCBBuildServiceProxy",
+                "XCBProtocol",
+                "XCBProtocol_13_0",
+            ],
+            path: "Sources"
+        ),
+    ]
+)

Examples/BazelProgressXCBBuildService/Sources/BazelBuild.swift

@@ -0,0 +1,113 @@
+import Foundation
+import XCBBuildServiceProxy
+import XCBProtocol
+@_exported import XCBProtocol_13_0
+
+// swiftformat:disable braces
+
+final class BazelBuild {
+
+    private let buildContext: BuildContext<BazelXCBBuildServiceResponsePayload>
+    private let buildProcess: BazelBuildProcess
+
+    private var buildProgress: Double = -1.0
+    private var initialActionCount: Int = 0
+    private var totalActions: Int = 0
+    private var completedActions: Int = 0
+
+    /// This regex is used to minimally remove the timestamp at the start of our messages.
+    /// After that we try to parse out the execution progress
+    /// (see https://github.com/bazelbuild/bazel/blob/9bea69aee3acf18b780b397c8c441ac5715d03ae/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionProgressReceiver.java#L150-L157 ).
+    /// Finally we throw away any " ... (8 actions running)" like messages (see https://github.com/bazelbuild/bazel/blob/4f0b710e2b935b4249e0bbf633f43628bbf93d7b/src/main/java/com/google/devtools/build/lib/runtime/UiStateTracker.java#L1158 ).
+    private static let progressRegex = try! NSRegularExpression(
+        pattern: #"^(?:\(\d{1,2}:\d{1,2}:\d{1,2}\) )?(?:\[(\d{1,3}(,\d{3})*) \/ (\d{1,3}(,\d{3})*)\] )?(?:(?:INFO|ERROR|WARNING): )?(.*?)(?: \.\.\. \(.*\))?$"#
+    )
+
+    init(buildContext: BuildContext<BazelXCBBuildServiceResponsePayload>) throws {
+        self.buildContext = buildContext
+        self.buildProcess = BazelClient()
+    }
+
+    func start() throws {
+        try buildProcess.start(
+            bepHandler: { [buildContext] event in
+                var progressMessage: String?
+                event.progress.stdout.split(separator: "\n").forEach { message in
+                    guard !message.isEmpty else { return }
+
+                    let message = String(message)
+                    logger.info("message out: \(message)")
+                }
+
+                event.progress.stderr.split(separator: "\n").forEach { message in
+                    guard !message.isEmpty else { return }
+
+                    let message = String(message)
+                    logger.info("message err: \(message)")
+
+                    if
+                        let match = Self.progressRegex.firstMatch(
+                            in: message,
+                            options: [],
+                            range: NSRange(message.startIndex ..< message.endIndex, in: message)
+                        ),
+                        match.numberOfRanges == 6,
+                        let finalMessageRange = Range(match.range(at: 5), in: message),
+                        let completedActionsRange = Range(match.range(at: 1), in: message),
+                        let totalActionsRange = Range(match.range(at: 3), in: message)
+                    {
+                        progressMessage = String(message[finalMessageRange]).components(separatedBy: ";").first
+
+                        let completedActionsString = message[completedActionsRange]
+                            .replacingOccurrences(of: ",", with: "")
+                        let totalActionsString = message[totalActionsRange]
+                            .replacingOccurrences(of: ",", with: "")
+
+                        if
+                            let completedActions = Int(completedActionsString),
+                            let totalActions = Int(totalActionsString)
+                        {
+                            self.totalActions = totalActions
+                            self.completedActions = completedActions
+                            if self.initialActionCount == 0, completedActions > 0, completedActions != totalActions {
+                                self.initialActionCount = completedActions
+                            }
+
+                            self.buildProgress = 100 * Double(completedActions - self.initialActionCount) / Double(totalActions - self.initialActionCount)
+                        } else {
+                            logger.error("Failed to parse progress out of BEP message: \(message)")
+                        }
+                    }
+                }
+
+                if event.lastMessage {
+                    progressMessage = progressMessage ?? "Compilation complete"
+                    self.buildProgress = 100
+                }
+
+                // Take the last message in the case of multiple lines, as well as the most recent `buildProgress`
+                if let message = progressMessage {
+                    buildContext.progressUpdate("\(message) \(self.completedActions)/\(self.totalActions)", percentComplete: self.buildProgress)
+                }
+            }
+        )
+    }
+
+    func cancel() {
+        buildProcess.stop()
+    }
+}
+
+
+private extension BuildContext where ResponsePayload == BazelXCBBuildServiceResponsePayload {
+    func progressUpdate(_ message: String, percentComplete: Double, showInLog: Bool = false) {
+        sendResponseMessage(
+            BuildOperationProgressUpdated(
+                targetName: nil,
+                statusMessage: message,
+                percentComplete: percentComplete,
+                showInLog: showInLog
+            )
+        )
+    }
+}

Examples/BazelProgressXCBBuildService/Sources/BazelBuildProcess.swift

@@ -0,0 +1,175 @@
+import Foundation
+import src_main_java_com_google_devtools_build_lib_buildeventstream_proto_build_event_stream_proto
+import SwiftProtobuf
+
+protocol BazelBuildProcess {
+    func start(bepHandler: @escaping (BuildEventStream_BuildEvent) -> Void) throws
+    func stop()
+}
+
+enum BazelBuildProcessError: Error {
+    case alreadyStarted
+    case failedToCreateBEPFile
+}
+
+/// Encapsulates a child Bazel script process.
+final class BazelClient: BazelBuildProcess {
+    /// Queue used to ensure proper ordering of results from process output/termination.
+    private let processResultsQueue = DispatchQueue(
+        label: "BazelXCBBuildService.BazelBuildProcess",
+        qos: .userInitiated
+    )
+
+    private var isRunning = false
+    private var isCancelled = false
+    private let process: Process
+    private let bepPath: String
+
+    init() {
+        //RAPPI: We use the same BEP path as XCBuildKit
+        self.bepPath = "/tmp/bep.bep"
+        print("BEP Path: \(self.bepPath)")
+        self.process = Process()
+
+        // Automatically terminate process if our process exits
+        let selector = Selector(("setStartsNewProcessGroup:"))
+        if process.responds(to: selector) {
+            process.perform(selector, with: false as NSNumber)
+        }
+    }
+
+    func start(bepHandler: @escaping (BuildEventStream_BuildEvent) -> Void) throws {
+        guard !process.isRunning else {
+            throw BazelBuildProcessError.alreadyStarted
+        }
+
+        let fileManager = FileManager.default
+
+        fileManager.createFile(atPath: bepPath, contents: Data())
+        guard let bepFileHandle = FileHandle(forReadingAtPath: bepPath) else {
+            logger.error("Failed to create file for BEP stream at “\(bepPath)”")
+            throw BazelBuildProcessError.failedToCreateBEPFile
+        }
+
+        /// Dispatch group used to ensure that stdout and stderr are processed before the process termination.
+        /// This is needed since all three notifications come in on different threads.
+        let processDispatchGroup = DispatchGroup()
+
+        /// `true` if the `terminationHandler` has been called. Xcode will crash if we send more events after that.
+        var isTerminated = false
+
+        // Bazel works by appending content to a file, specifically, Java's `BufferedOutputStream`.
+        // Naively using an input stream for the path and waiting for available data simply does not work with
+        // whatever `BufferedOutputStream.flush()` is doing internally.
+        //
+        // Reference:
+        // https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java
+        //
+        // Perhaps, SwiftProtobuf can come up with a better solution to read from files or upstream similar code:
+        // https://github.com/apple/swift-protobuf/issues/130
+        //
+        // Logic:
+        // - Create a few file
+        // - When the build starts, Bazel will attempt to reuse the inode, and stream to it
+        // - Then, via `FileHandle`, wait for data to be available and read all the bytes
+        bepFileHandle.readabilityHandler = { [processResultsQueue] _ in
+            // `bepFileHandle` is captured in the closure, which keeps the reference around
+            let data = bepFileHandle.availableData
+            guard !data.isEmpty else {
+                return
+            }
+
+            processDispatchGroup.enter()
+            processResultsQueue.async {
+                defer { processDispatchGroup.leave() }
+
+                // We don't want to report any more progress if the build has been terminated
+                guard !isTerminated else {
+                    bepFileHandle.closeFile()
+                    bepFileHandle.readabilityHandler = nil
+                    return
+                }
+
+                // Wrap the file handle in an `InputStream` for SwiftProtobuf to read
+                // We read the stream until the (current) end of the file
+                let input = InputStream(data: data)
+                input.open()
+                while input.hasBytesAvailable {
+                    do {
+                        let event = try BinaryDelimited.parse(messageType: BuildEventStream_BuildEvent.self, from: input)
+
+                        logger.trace("Received BEP event: \(event)")
+
+                        bepHandler(event)
+
+                        if event.lastMessage {
+                            logger.trace("Received last BEP event")
+
+                            bepFileHandle.closeFile()
+                            bepFileHandle.readabilityHandler = nil
+                        }
+                    } catch {
+                        logger.error("Failed to parse BEP event: \(error)")
+                        return
+                    }
+                }
+            }
+        }
+
+        let stdout = Pipe()
+        let stderr = Pipe()
+
+        processDispatchGroup.enter()
+        stdout.fileHandleForReading.readabilityHandler = { [processResultsQueue] handle in
+            let data = handle.availableData
+            guard !data.isEmpty else {
+                logger.trace("Received Bazel standard output EOF")
+                stdout.fileHandleForReading.readabilityHandler = nil
+                processDispatchGroup.leave()
+
+                return
+            }
+
+            processResultsQueue.async {
+                logger.trace("Received Bazel standard output: \(data)")
+            }
+        }
+
+        processDispatchGroup.enter()
+        stderr.fileHandleForReading.readabilityHandler = { [processResultsQueue] handle in
+            let data = handle.availableData
+            guard !data.isEmpty else {
+                logger.trace("Received Bazel standard error EOF")
+                stderr.fileHandleForReading.readabilityHandler = nil
+                processDispatchGroup.leave()
+                return
+            }
+
+            processResultsQueue.async {
+                logger.trace("Received Bazel standard error: \(data)")
+            }
+        }
+
+        process.standardOutput = stdout
+        process.standardError = stderr
+
+        processDispatchGroup.enter()
+        process.terminationHandler = { process in
+            logger.debug("xcode.sh exited with status code: \(process.terminationStatus)")
+            processDispatchGroup.leave()
+        }
+
+        processDispatchGroup.notify(queue: processResultsQueue) {
+            logger.info("\(self.isCancelled ? "Cancelled Bazel" : "Bazel") build exited with status code: \(self.process.terminationStatus)")
+            isTerminated = true
+        }
+    }
+
+    func stop() {
+        isCancelled = true
+        if process.isRunning {
+            // Sends SIGTERM to the Bazel client. It will cleanup and exit.
+            process.terminate()
+        }
+    }
+}