areusch commented on a change in pull request #6334: URL: https://github.com/apache/incubator-tvm/pull/6334#discussion_r485070810
########## File path: include/tvm/runtime/crt/utvm_rpc_server.h ########## @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file utvm_rpc_server.h + * \brief MicroTVM RPC Server + */ + +#ifndef TVM_RUNTIME_CRT_UTVM_RPC_SERVER_H_ +#define TVM_RUNTIME_CRT_UTVM_RPC_SERVER_H_ + +#include <stdlib.h> +#include <sys/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/*! \brief TVM RPC channel write function. + * + * Tries to write `num_bytes` from `data` to the underlying channel. + * \param data Pointer to data to write. + * \param num_bytes Number of bytes avaiable in data. + * \return The number of bytes written. + */ +typedef ssize_t (*utvm_rpc_channel_write_t)(void* context, const uint8_t* data, size_t num_bytes); + +/*! \brief Opaque pointer type to TVM RPC Server. */ +typedef void* utvm_rpc_server_t; + +/*! \brief Initialize the TVM RPC Server. + * + * Call this on device startup before calling anyother utvm_rpc_server_ functions. + * + * \param memory A memory block used by the runtime as dynamic memory, primarily to allocate + * tensors. + * \param memory_size_bytes Size of the memory block, in bytes. Should be a multiple of + * (1 << page_size_bytes_log2) + * \param page_size_bytes_log2 Log2 of the size of each memory page. The internal allocator + * allocates one page at a time; more pages reduces waste but + * increases overhead. + * \param write_func A callback function invoked by the TVM RPC Server to write data back to the + * host. Internally, the TVM RPC Server will block until all data in a reply + * packet has been written. + * \param write_func_ctx An opaque pointer passed to write_func when it is called. + * \return A pointer to the TVM RPC Server. The pointer is allocated in the same memory space as + * the TVM workspace. + */ +utvm_rpc_server_t utvm_rpc_server_init(uint8_t* memory, size_t memory_size_bytes, Review comment: done ########## File path: include/tvm/runtime/crt/rpc_common/session.h ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file session.h + * \brief RPC Session + */ + +#ifndef TVM_RUNTIME_CRT_RPC_COMMON_SESSION_H_ +#define TVM_RUNTIME_CRT_RPC_COMMON_SESSION_H_ + +#include <inttypes.h> +#include <tvm/runtime/crt/error_codes.h> +#include <tvm/runtime/crt/rpc_common/buffer.h> +#include <tvm/runtime/crt/rpc_common/framing.h> +#include <tvm/runtime/crt/rpc_common/write_stream.h> + +namespace tvm { +namespace runtime { + +enum class MessageType : uint8_t { + kStartSessionInit = 0x00, + kStartSessionReply = 0x01, + kTerminateSession = 0x02, + kLog = 0x03, + kNormal = 0x10, +}; + +typedef struct SessionHeader { + uint16_t session_id; + MessageType message_type; +} __attribute__((packed)) SessionHeader; + +/*! + * \brief CRT communication session management class. + * Assumes the following properties provided by the underlying transport: + * - in-order delivery. + * - reliable delivery. + * + * Specifically, designed for use with UARTs. Will probably work over semihosting, USB, and TCP; + * will probably not work reliably enough over UDP. + */ +class Session { + public: + /*! \brief Callback invoked when a full message is received. + * + * This function is called in the following situations: + * - When a new session is established (this typically indicates the remote end reset). + * In this case, buf is NULL. + * - When a log message or normal traffic is received. In this case, buf points to a + * valid buffer containing the message content. + * + * \param context The value of `message_received_func_context` passed to the constructor. + * \param message_type The type of session message received. Currently, this is always + * either kNormal or kLog. + * \param buf When message_type is not kStartSessionMessage, a Buffer whose read cursor is at the + * first byte of the message payload. Otherwise, NULL. + */ + typedef void (*MessageReceivedFunc)(void* context, MessageType message_type, Buffer* buf); + + /*! \brief An invalid nonce value that typically indicates an unknown nonce. */ + static constexpr const uint8_t kInvalidNonce = 0; + + Session(uint8_t initial_session_nonce, Framer* framer, Buffer* receive_buffer, + MessageReceivedFunc message_received_func, void* message_received_func_context) + : local_nonce_{initial_session_nonce}, + session_id_{0}, + state_{State::kReset}, + receiver_{this}, + framer_{framer}, + receive_buffer_{receive_buffer}, + receive_buffer_has_complete_message_{false}, + message_received_func_{message_received_func}, + message_received_func_context_{message_received_func_context} { + // Session can be used for system startup logging, before the RPC server is instantiated. In + // this case, allow receive_buffer_ to be nullptr. The instantiator agrees not to use + // Receiver(). + if (receive_buffer_ != nullptr) { + receive_buffer_->Clear(); + } + } + + /*! + * \brief Send a session terminate message, usually done at startup to interrupt a hanging remote. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t Initialize(); + + /*! + * \brief Terminate any previously-established session. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t TerminateSession(); + + /*! + * \brief Start a new session regardless of state. Sends kStartSessionMessage. + * + * Generally speaking, this function should be called once per device reset by exactly one side + * in the system. No traffic can flow until this function is called. + * + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t StartSession(); + + /*! + * \brief Obtain a WriteStream implementation for use by the framing layer. + * \return A WriteStream to which received data should be written. Owned by this class. + */ + WriteStream* Receiver() { return &receiver_; } + + /*! + * \brief Send a full message including header, payload, and CRC footer. + * \param message_type One of MessageType; distinguishes the type of traffic at the session layer. + * \param message_data The data contained in the message. + * \param message_size_bytes The number of valid bytes in message_data. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t SendMessage(MessageType message_type, const uint8_t* message_data, + size_t message_size_bytes); + + /*! + * \brief Send the framing and session layer headers. + * + * This function allows messages to be sent in pieces. + * + * \param message_type One of MessageType; distinguishes the type of traffic at the session layer. + * \param message_size_bytes The size of the message body, in bytes. Excludes the framing and + * session layer headers. \return 0 on success, negative error code on failure. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t StartMessage(MessageType message_type, size_t message_size_bytes); + + /*! + * \brief Send a part of the message body. + * + * This function allows messages to be sent in pieces. + * + * \param chunk_data The data contained in this message body chunk. + * \param chunk_size_bytes The number of valid bytes in chunk_data. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t SendBodyChunk(const uint8_t* chunk_data, size_t chunk_size_bytes); + + /*! + * \brief Finish sending the message by sending the framing layer footer. + * \return kTvmErrorNoError on success, or an error code otherwise. + */ + tvm_crt_error_t FinishMessage(); + + /*! \brief Returns true if the session is in the established state. */ + bool IsEstablished() const { return state_ == State::kSessionEstablished; } + + /*! + * \brief Clear the receive buffer and prepare to receive next message. + * + * Call this function after MessageReceivedFunc is invoked. Any SessionReceiver::Write() calls + * made will return errors until this function is called to prevent them from corrupting the + * valid message in the receive buffer. + */ + void ClearReceiveBuffer(); + + private: + class SessionReceiver : public WriteStream { + public: + explicit SessionReceiver(Session* session) : session_{session} {} + virtual ~SessionReceiver() {} + + ssize_t Write(const uint8_t* data, size_t data_size_bytes) override; + void PacketDone(bool is_valid) override; + + private: + void operator delete(void*) noexcept {} // NOLINT(readability/casting) + Session* session_; + }; + + enum class State : uint8_t { + kReset = 0, + kNoSessionEstablished = 1, + kStartSessionSent = 2, + kSessionEstablished = 3, + }; + + void RegenerateNonce(); + + tvm_crt_error_t SendInternal(MessageType message_type, const uint8_t* message_data, + size_t message_size_bytes); + + void SendSessionStartReply(const SessionHeader& header); + + void ProcessStartSessionInit(const SessionHeader& header); + + void ProcessStartSessionReply(const SessionHeader& header); + + void OnSessionEstablishedMessage(); + + void OnSessionTerminatedMessage(); + + inline void SetSessionId(uint8_t initiator_nonce, uint8_t responder_nonce) { + session_id_ = initiator_nonce | (((uint16_t)responder_nonce) << 8); + } + + inline uint8_t initiator_nonce(uint16_t session_id) { return session_id & 0xff; } Review comment: done ########## File path: python/tvm/contrib/graph_runtime.py ########## @@ -43,24 +43,34 @@ def create(graph_json_str, libmod, ctx): be used as this purpose. All context should be given for heterogeneous execution. + force_local_graph_runtime : bool + When ctx contains only rpc contexts, the GraphRuntime is typically instantiated Review comment: I guess i'm ambivalent here; a micro-specific e.g. `tvm.micro.Session.create_graph_runtime()` would have a pretty minimal implementation, but it would create a second Python counterpart to `tvm.graph_runtime.create`, so maintenance could be more complex (I.e. you have to know to read `python/tvm/micro/session.py` to understand all the possible use cases of the C++ graph runtime. it's true that micro is the only thing using this code path, but it also is reusable in any RPC context. it's also not clear to me the future of the graph runtime--why is there the prohibition on mixing local and remote contexts? ########## File path: include/tvm/runtime/crt/rpc_common/session.h ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file session.h + * \brief RPC Session + */ + +#ifndef TVM_RUNTIME_CRT_RPC_COMMON_SESSION_H_ +#define TVM_RUNTIME_CRT_RPC_COMMON_SESSION_H_ + +#include <inttypes.h> +#include <tvm/runtime/crt/error_codes.h> +#include <tvm/runtime/crt/rpc_common/buffer.h> +#include <tvm/runtime/crt/rpc_common/framing.h> +#include <tvm/runtime/crt/rpc_common/write_stream.h> + +namespace tvm { +namespace runtime { + +enum class MessageType : uint8_t { + kStartSessionInit = 0x00, + kStartSessionReply = 0x01, + kTerminateSession = 0x02, + kLog = 0x03, + kNormal = 0x10, +}; + +typedef struct SessionHeader { + uint16_t session_id; + MessageType message_type; +} __attribute__((packed)) SessionHeader; + +/*! + * \brief CRT communication session management class. + * Assumes the following properties provided by the underlying transport: + * - in-order delivery. + * - reliable delivery. + * + * Specifically, designed for use with UARTs. Will probably work over semihosting, USB, and TCP; + * will probably not work reliably enough over UDP. + */ +class Session { Review comment: moved to `tvm::runtime::micro_rpc` ########## File path: include/tvm/runtime/crt/rpc_common/framing.h ########## @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file framing.h + * \brief Framing for RPC. + */ + +#ifndef TVM_RUNTIME_CRT_RPC_COMMON_FRAMING_H_ +#define TVM_RUNTIME_CRT_RPC_COMMON_FRAMING_H_ + +#include <crc16.h> +#include <inttypes.h> +#include <stddef.h> +#include <tvm/runtime/crt/error_codes.h> +#include <tvm/runtime/crt/rpc_common/write_stream.h> + +namespace tvm { +namespace runtime { + +enum class Escape : uint8_t { kEscapeStart = 0xff, kEscapeNop = 0xfe, kPacketStart = 0xfd }; + +class PacketFieldSizeBytes { + public: + static constexpr const size_t kPayloadLength = sizeof(uint32_t); + static constexpr const size_t kCrc = sizeof(uint16_t); +}; + +class Unframer { + public: + explicit Unframer(WriteStream* stream) + : stream_{stream}, + state_{State::kFindPacketStart}, + saw_escape_start_{false}, + num_buffer_bytes_valid_{0} {} + + /*! + * \brief Push data into unframer and try to decode one packet. + * + * This function will return when exactly one packet has been decoded. It may not consume all of + * `data` in this case, and valid bytes may remain at the end of data. + * + * \param data The new data to unframe and send downstream. + * \param data_size_bytes The number of valid bytes in data. + * \param bytes_consumed Pointer written with the number of bytes consumed from data. + * \return + * - kTvmErrorNoError when successful -- continue writing data. + * - kTvmErrorFramingInvalidState when the Unframer was in or enters an invalid state + * (probably indicates memory corruption). + * - kTvmErrorFramingShortPacket when a new packet started before the current one ended. + * - kTvmErrorFramingInvalidEscape when an invalid escape sequence was seen + */ + tvm_crt_error_t Write(const uint8_t* data, size_t data_size_bytes, size_t* bytes_consumed); + + /*! \brief Reset unframer to initial state. */ + void Reset(); + + /*! \brief Return an underestimate of the number of bytes needed from the wire. */ + size_t BytesNeeded(); + + private: + tvm_crt_error_t FindPacketStart(); + tvm_crt_error_t FindPacketLength(); + tvm_crt_error_t FindPacketCrc(); + tvm_crt_error_t FindCrcEnd(); + + inline bool is_buffer_full(size_t buffer_full_bytes) { + return num_buffer_bytes_valid_ >= buffer_full_bytes; + } + + /*! \brief Consume input into buffer_ until buffer_ has buffer_full_bytes. */ + tvm_crt_error_t AddToBuffer(size_t buffer_full_bytes, bool update_crc); + + void ClearBuffer(); + + /*! \brief Unescape and consume input bytes, storing into buffer. + * + * \param buffer A buffer to fill with consumed, unescaped bytes. + * \param buffer_size_bytes Size of buffer, in bytes. + * \param bytes_filled A pointer to an accumulator to which is added the number of bytes written + * to `buffer`. + * \param update_crc true when the CRC should be updated with the escaped bytes. + * \return + * - kTvmErrorNoError if successful + * - kTvmErrorFramingShortPacket if a start-of-packet escape code was encountered. If so, + * *bytes_filled indicates the number of bytes before the Escape::kEscapeStart byte. + * - kTvmErrorFramingInvalidEscape if an invalid escape sequence was seen. + * - kTvmErrorWriteStreamShortWrite if the WriteStream passed to constructor's Write() + * function returns 0. + * - kTvmErrorWriteStreamShortWrite if the WriteStream passed to constructor's Write() + * function returns an invalid positive number. + * - Any negative value (i.e. with bits in kTvmErrorSystemErrorMask set) returned by the + * WriteStream's Write() function. + */ Review comment: I guess the thinking was that currently these components are defined in the CRT, and the C++ runtime is external to that. we could also move the logic in `src/runtime/micro/micro_session.cc` into e.g. `src/runtime/crt/rpc_client` or some other location inside CRT, then export a slimmer public API for that; this could allow other languages to interact with the RPC server to build e.g. slimmer debug tooling. thoughts? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
