todo: iocp recv안되는 문제 해결하기

include/socket/address.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include <openssl/bio.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
 namespace Network {
 
 struct Address {
@@ -8,9 +12,6 @@ struct Address {
 
   void zeroFill();
   void set(int type, gsl::czstring presentationAddr, std::uint16_t port);
-  void set(int type, in_addr_t addr, std::uint16_t port);
-  void set(int type, in_addr addr, std::uint16_t port);
-  void set(int type, in6_addr addr, std::uint16_t port);
   void setType(int type);
 
   operator std::string();
@@ -23,6 +24,8 @@ struct Address {
     struct sockaddr_in6 addr_in6;
   };
   socklen_t length;
+
+  BIO_ADDR* bio_addr_info;
 };
 
 }  // namespace Chattr

include/socket/iocp.h

@@ -6,6 +6,7 @@
 #include <functional>
 #include <queue>
 #include <vector>
+#include <random>
 
 #include "socket.h"
 #include "utils/thread_pool.h"
@@ -30,7 +31,7 @@ class IOCP;
 
 enum class IOCPEVENT { QUIT, READ, WRITE };
 
-struct IOCPPASSINDATA {
+struct IOCPPASSINDATA { // 얘 double free 문제 있음..
   OVERLAPPED overlapped;
   IOCPEVENT event;
   std::shared_ptr<Socket> socket;
@@ -49,6 +50,8 @@ struct IOCPPASSINDATA {
     std::memset(&overlapped, 0, sizeof(overlapped));
     event = IOCPEVENT::QUIT;
     socket = nullptr;
+    rbio = nullptr;
+    wbio = nullptr;
     transferredbytes = 0;
     this->bufsize = bufsize;
     IOCPInstance = nullptr;
@@ -56,25 +59,44 @@ struct IOCPPASSINDATA {
     wsabuf.buf = new char[bufsize];
     wsabuf.len = bufsize;
   }
-  IOCPPASSINDATA(const IOCPPASSINDATA& other)
-      : event(other.event),
-        socket(other.socket),
-        transferredbytes(other.transferredbytes),
-        bufsize(other.bufsize),
-        IOCPInstance(other.IOCPInstance)
-#ifdef __linux__
-        ,
-        sendQueue(other.sendQueue)
-#endif
-  {
+  IOCPPASSINDATA(std::uint32_t bufsize, SSL_CTX* ctx) {
     std::memset(&overlapped, 0, sizeof(overlapped));
-    wsabuf.buf = new char[other.bufsize];
-    wsabuf.len = other.bufsize;
-    std::memcpy(wsabuf.buf, other.wsabuf.buf, other.wsabuf.len);
+    event = IOCPEVENT::QUIT;
+    socket = nullptr;
+    ssl = ::SSL_new(ctx);
+    rbio = ::BIO_new(::BIO_s_mem());
+    wbio = ::BIO_new(::BIO_s_mem());
+    ::SSL_set_bio(ssl, rbio, wbio);
+    transferredbytes = 0;
+    this->bufsize = bufsize;
+    IOCPInstance = nullptr;
+
+    wsabuf.buf = new char[bufsize];
+    wsabuf.len = bufsize;
+  }
+
+  IOCPPASSINDATA(const IOCPPASSINDATA& other) {
+    if (this != &other) {
+      std::memset(&overlapped, 0, sizeof(overlapped));
+      event = other.event;
+      socket = other.socket;
+      rbio = other.rbio;
+      wbio = other.wbio;
+      transferredbytes = other.transferredbytes;
+      bufsize = other.bufsize;
+      IOCPInstance = other.IOCPInstance;
+#ifdef __linux__
+      sendQueue = other.sendQueue;
+#endif
+      wsabuf.buf = new char[other.bufsize];
+      wsabuf.len = other.bufsize;
+      std::memcpy(wsabuf.buf, other.wsabuf.buf, other.wsabuf.len);
+    }
   }
 
   ~IOCPPASSINDATA() {
     if (wsabuf.buf != nullptr) delete[] wsabuf.buf;
+    wsabuf.buf = nullptr;
   }
 
   IOCPPASSINDATA& operator=(const IOCPPASSINDATA& other) {
@@ -82,6 +104,8 @@ struct IOCPPASSINDATA {
       std::memset(&overlapped, 0, sizeof(overlapped));
       event = other.event;
       socket = other.socket;
+      rbio = other.rbio;
+      wbio = other.wbio;
       transferredbytes = other.transferredbytes;
       bufsize = other.bufsize;
       IOCPInstance = other.IOCPInstance;
@@ -99,16 +123,13 @@ struct IOCPPASSINDATA {
 
 class IOCP {
  public:
+  IOCP();
   ~IOCP();
 
   void init(utils::ThreadPool* __IOCPThread, SessionProtocol proto) {
     IOCPThread_ = __IOCPThread;
     proto_ = proto;
 
-    if (proto == SessionProtocol::TLS || proto == SessionProtocol::QUIC) {
-      rbio_ = ::BIO_new(::BIO_s_mem());
-      wbio_ = ::BIO_new(::BIO_s_mem());
-    }
 #ifdef _WIN32
     completionPort_ =
         ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
@@ -134,8 +155,8 @@ class IOCP {
   void registerSocket(IOCPPASSINDATA* data);
 
   // data는 한 가지 소켓에 보내는 패킷만 담아야 합니다
-  int send(SOCKET sock, std::vector<IOCPPASSINDATA>& data);
-  int recv(IOCPPASSINDATA& data);
+  int send(SOCKET sock, std::vector<IOCPPASSINDATA*>* data);
+  int recv(IOCPPASSINDATA* data);
 
  private:
 #ifdef _WIN32
@@ -143,11 +164,19 @@ class IOCP {
     IOCPPASSINDATA* data;
     SOCKET sock;
     DWORD cbTransfrred;
+    int jitter = jitterDist_(gen_);
     int retVal = GetQueuedCompletionStatus(completionPort_, &cbTransfrred,
                                            (PULONG_PTR)&sock,
-                                           (LPOVERLAPPED*)&data, INFINITE);
+                                           (LPOVERLAPPED*)&data, 1000 + jitter);
 
     if (retVal == 0 || cbTransfrred == 0) {
+      DWORD lasterror = GetLastError();
+      if (lasterror == WAIT_TIMEOUT) {
+        IOCPThread->enqueueJob([this](utils::ThreadPool* th,
+                                      std::uint8_t __) { iocpWatcher_(th); },
+                               0);
+        return;
+      }
       data->event = IOCPEVENT::QUIT;
       spdlog::debug("Disconnected. [{}]",
                     (std::string)(data->socket->remoteAddr));
@@ -161,7 +190,7 @@ class IOCP {
     auto queue_list = GetRecvQueue_(data->socket->sock);
     if (data->event == IOCPEVENT::READ) {
       if (proto_ == SessionProtocol::TLS || proto_ == SessionProtocol::QUIC) {
-        ::BIO_write(rbio_, data->wsabuf.buf, cbTransfrred);
+        ::BIO_write(data->rbio, data->wsabuf.buf, cbTransfrred);
 
         while ((red_data = ::SSL_read(data->ssl, buf.data(), buf.size())) > 0) {
           queue_list->emplace_back(std::make_pair(
@@ -188,22 +217,21 @@ class IOCP {
 
 #endif
 
-  std::shared_ptr<std::list<IOCPPASSINDATA>> GetSendQueue_(SOCKET sock);
+  std::shared_ptr<std::list<IOCPPASSINDATA*>> GetSendQueue_(SOCKET sock);
   std::shared_ptr<std::list<std::pair<std::vector<char>, std::uint32_t>>>
   GetRecvQueue_(SOCKET sock);
   std::shared_ptr<std::mutex> GetSendQueueMutex_(SOCKET sock);
   std::shared_ptr<std::mutex> GetRecvQueueMutex_(SOCKET sock);
 
   void packet_sender_(SOCKET sock);
-
-  struct WSAManager* wsaManager = WSAManager::GetInstance();
   utils::ThreadPool* IOCPThread_;
 
-  BIO* rbio_ = nullptr;
-  BIO* wbio_ = nullptr;
-
   SessionProtocol proto_;
 
+  std::random_device rd_;
+  std::mt19937 gen_;
+  std::uniform_int_distribution<int> jitterDist_;
+
   // 밑의 unordered_map들에 키를 추가/제거 하려는 스레드는 이 뮤텍스를 잡아야
   // 함.
   std::mutex socket_mod_mutex_;
@@ -233,7 +261,7 @@ class IOCP {
   // 스레드가 데이터를 추가하고 미전송된 경우를 대비하여 스레드가 대기하도록
   // 한다. 리눅스에서도 send_queue에 데이터를 쌓고 스레드가 대기하도록 한다.
   std::unordered_map<SOCKET, std::shared_ptr<std::mutex>> send_queue_mutex_;
-  std::unordered_map<SOCKET, std::shared_ptr<std::list<IOCPPASSINDATA>>>
+  std::unordered_map<SOCKET, std::shared_ptr<std::list<IOCPPASSINDATA*>>>
       send_queue_;
   // 쓰기 싫었지만 쓰기 큐를 직렬화 하는 것 밖에 좋은 수가 생각이 안 남..
   /*std::mutex send_queue_mutex_;

include/socket/wsa_manager.h

@@ -4,15 +4,6 @@ namespace Network {
 
 struct WSAManager {
  public:
-  static WSAManager* GetInstance() {
-    static WSAManager instance;
-    return &instance;
-  }
-
-  WSAManager(const WSAManager&) = delete;
-  WSAManager& operator=(const WSAManager&) = delete;
-
- private:
 	WSAManager();
 	~WSAManager();
 };

include/utils/thread_pool.h

@@ -12,11 +12,9 @@ namespace utils {
 
 class ThreadPool {
  public:
-  ThreadPool();
   ThreadPool(std::uint32_t numThreads);
   ~ThreadPool();
 
-  void init(std::uint32_t numThreads);
   void terminate();
 
   void respawnWorker(std::uint32_t numThreads);
@@ -66,6 +64,8 @@ class ThreadPool {
   int threadCount = 0;
 
  private:
+  void init_(std::uint32_t numThreads);
+
   void* Worker();
 
   std::condition_variable jobQueueCV_;