windows-nt/Source/XPSP1/NT/base/cluster/clusrtl/threadq.c

639 lines
17 KiB
C
Raw Normal View History

2020-09-26 03:20:57 -05:00
/*++
Copyright (c) 1996 Microsoft Corporation
Module Name:
threadq.c
Abstract:
Generic Worker Thread Queue Package
Author:
Mike Massa (mikemas) April 5, 1996
Revision History:
Who When What
-------- -------- ----------------------------------------------
mikemas 04-05-96 created
Notes:
Worker Thread Queues provide a single mechanism for processing
overlapped I/O completions as well as deferred work items. Work
queues are created and destroyed using ClRtlCreateWorkQueue()
and ClRtlDestroyWorkQueue(). Overlapped I/O completions are
directed to a work queue by associating an I/O handle with a work
queue using ClRtlAssociateIoHandleWorkQueue(). Deferred work items
are posted to a work queue using ClRtlPostItemWorkQueue().
Work queues are implemented using I/O completion ports. Each work
queue is serviced by a set of threads which dispatch work items
to specified work routines. Threads are created dynamically, up to
a specified maximum, to ensure that there is always a thread waiting
to service new work items. The priority of the threads servicing a
work queue can be specified.
[Future enhancement: dynamically shrink the thread pool when the
incoming work rate drops off. Currently, threads continue to service
the work queue until it is destroyed.]
Special care must be taken when destroying a work queue to ensure
that all threads terminate properly and no work items are lost.
See the notes under ClRtlDestroyWorkQueue.
--*/
#include "clusrtlp.h"
//
// Private Types
//
typedef struct _CLRTL_WORK_QUEUE {
HANDLE IoCompletionPort;
LONG MaximumThreads;
LONG TotalThreads;
LONG WaitingThreads;
LONG ReserveThreads;
LONG ConcurrentThreads;
DWORD Timeout;
int ThreadPriority;
HANDLE StopEvent;
} CLRTL_WORK_QUEUE;
//
// Private Routines
//
DWORD
ClRtlpWorkerThread(
LPDWORD Context
)
{
PCLRTL_WORK_QUEUE workQueue = (PCLRTL_WORK_QUEUE) Context;
DWORD bytesTransferred;
BOOL ioSuccess;
ULONG_PTR ioContext;
LPOVERLAPPED overlapped;
PCLRTL_WORK_ITEM workItem;
DWORD status;
LONG interlockedResult;
DWORD threadId;
HANDLE threadHandle;
DWORD timeout;
LONG myThreadId;
timeout = workQueue->Timeout;
myThreadId = GetCurrentThreadId();
if (!SetThreadPriority(GetCurrentThread(), workQueue->ThreadPriority)) {
status = GetLastError();
ClRtlLogPrint(
LOG_UNUSUAL,
"[WTQ] Thread %1!u! unable to set priority to %2!d!, status %3!u!\n",
myThreadId,
workQueue->ThreadPriority,
status
);
}
#if THREADQ_VERBOSE
ClRtlLogPrint(
LOG_CRITICAL,
"[WTQ] Thread %1!u! started, queue %2!lx!.\n",
myThreadId,
workQueue
);
#endif
while (TRUE) {
InterlockedIncrement(&(workQueue->WaitingThreads));
ioSuccess = GetQueuedCompletionStatus(
workQueue->IoCompletionPort,
&bytesTransferred,
&ioContext,
&overlapped,
timeout
);
interlockedResult = InterlockedDecrement(
&(workQueue->WaitingThreads)
);
if (overlapped) {
//
// Something was dequeued.
//
workItem = CONTAINING_RECORD(
overlapped,
CLRTL_WORK_ITEM,
Overlapped
);
if (interlockedResult == 0) {
//
// No more threads are waiting. Fire another one up.
// Make sure we haven't started too many first.
//
interlockedResult = InterlockedDecrement(
&(workQueue->ReserveThreads)
);
if (interlockedResult > 0) {
//
// We haven't started too many
//
#if THREADQ_VERBOSE
ClRtlLogPrint(
LOG_NOISE,
"[WTQ] Thread %1!u! starting another thread for queue %2!lx!.\n",
myThreadId,
workQueue
);
#endif // 0
InterlockedIncrement(&(workQueue->TotalThreads));
threadHandle = CreateThread(
NULL,
0,
ClRtlpWorkerThread,
workQueue,
0,
&threadId
);
if (threadHandle == NULL) {
InterlockedDecrement(&(workQueue->TotalThreads));
InterlockedIncrement(&(workQueue->ReserveThreads));
status = GetLastError();
ClRtlLogPrint(
LOG_CRITICAL,
"[WTQ] Thread %1!u! failed to create thread, %2!u!\n",
myThreadId,
status
);
}
else {
CloseHandle(threadHandle);
}
}
else {
InterlockedIncrement(&(workQueue->ReserveThreads));
}
} // end if (interlockedResult == 0)
if (ioSuccess) {
(*(workItem->WorkRoutine))(
workItem,
ERROR_SUCCESS,
bytesTransferred,
ioContext
);
}
else {
//
// The item was posted with an error.
//
status = GetLastError();
(*(workItem->WorkRoutine))(
workItem,
status,
bytesTransferred,
ioContext
);
}
continue;
}
else {
//
// No item was dequeued
//
if (ioSuccess) {
//
// This is our cue to start the termination process.
// Set the timeout to zero to make sure we don't block
// after the port is drained.
//
timeout = 0;
#if THREADQ_VERBOSE
ClRtlLogPrint(
LOG_NOISE,
"[WTQ] Thread %1!u! beginning termination process\n",
myThreadId
);
#endif // 0
}
else {
status = GetLastError();
if (status == WAIT_TIMEOUT) {
//
// No more items pending, time to exit.
//
CL_ASSERT(timeout == 0);
break;
}
CL_ASSERT(status == WAIT_TIMEOUT);
ClRtlLogPrint(
LOG_CRITICAL,
"[WTQ] Thread %1!u! No item, strange status %2!u! on queue %3!lx!\n",
myThreadId,
status,
workQueue
);
}
} // end if (overlapped)
} // end while(TRUE)
CL_ASSERT(workQueue->TotalThreads > 0);
InterlockedIncrement(&(workQueue->ReserveThreads));
InterlockedDecrement(&(workQueue->TotalThreads));
#if THREADQ_VERBOSE
ClRtlLogPrint(LOG_NOISE, "[WTQ] Thread %1!u! exiting.\n", myThreadId);
#endif // 0
//
// Let the ClRtlDestroyWorkQueue know we are terminating.
//
SetEvent(workQueue->StopEvent);
return(ERROR_SUCCESS);
}
//
// Public Routines
//
PCLRTL_WORK_QUEUE
ClRtlCreateWorkQueue(
IN DWORD MaximumThreads,
IN int ThreadPriority
)
/*++
Routine Description:
Creates a work queue and a dynamic pool of threads to service it.
Arguments:
MaximumThreads - The maximum number of threads to create to service
the queue.
ThreadPriority - The priority level at which the queue worker threads
should run.
Return Value:
A pointer to the created queue if the routine is successful.
NULL if the routine fails. Call GetLastError for extended
error information.
--*/
{
DWORD status;
PCLRTL_WORK_QUEUE workQueue = NULL;
DWORD threadId;
HANDLE threadHandle = NULL;
HANDLE bogusHandle = NULL;
if (MaximumThreads == 0) {
SetLastError(ERROR_INVALID_PARAMETER);
return(NULL);
}
bogusHandle = CreateFileW(
L"NUL",
GENERIC_READ,
FILE_SHARE_READ,
NULL,
OPEN_EXISTING,
FILE_FLAG_OVERLAPPED,
NULL
);
if (bogusHandle == INVALID_HANDLE_VALUE) {
status = GetLastError();
ClRtlLogPrint(LOG_CRITICAL, "[WTQ] bogus file creation failed, %1!u!\n", status);
return(NULL);
}
workQueue = LocalAlloc(
LMEM_FIXED | LMEM_ZEROINIT,
sizeof(CLRTL_WORK_QUEUE)
);
if (workQueue == NULL) {
status = ERROR_NOT_ENOUGH_MEMORY;
goto error_exit;
}
workQueue->MaximumThreads = MaximumThreads;
workQueue->TotalThreads = 1;
workQueue->WaitingThreads = 0;
workQueue->ReserveThreads = MaximumThreads - 1;
workQueue->ConcurrentThreads = 0;
workQueue->Timeout = INFINITE;
workQueue->ThreadPriority = ThreadPriority;
workQueue->IoCompletionPort = CreateIoCompletionPort(
bogusHandle,
NULL,
0,
workQueue->ConcurrentThreads
);
CloseHandle(bogusHandle); bogusHandle = NULL;
if (workQueue->IoCompletionPort == NULL) {
status = GetLastError();
ClRtlLogPrint(LOG_CRITICAL, "[WTQ] Creation of I/O Port failed, %1!u!\n", status);
}
workQueue->StopEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
if (workQueue->StopEvent == NULL) {
status = GetLastError();
ClRtlLogPrint(LOG_CRITICAL, "[WTQ] Creation of stop event failed, %1!u!\n", status);
goto error_exit;
}
threadHandle = CreateThread(
NULL,
0,
ClRtlpWorkerThread,
workQueue,
0,
&threadId
);
if (threadHandle == NULL) {
status = GetLastError();
ClRtlLogPrint(LOG_CRITICAL, "[WTQ] Failed to create worker thread, %1!u!\n", status);
goto error_exit;
}
CloseHandle(threadHandle);
return(workQueue);
error_exit:
if (bogusHandle != NULL) {
CloseHandle(bogusHandle);
}
if (workQueue != NULL) {
if (workQueue->IoCompletionPort != NULL) {
CloseHandle(workQueue->IoCompletionPort);
}
if (workQueue->StopEvent != NULL) {
CloseHandle(workQueue->StopEvent);
}
LocalFree(workQueue);
}
SetLastError(status);
return(NULL);
}
VOID
ClRtlDestroyWorkQueue(
IN PCLRTL_WORK_QUEUE WorkQueue
)
/*++
Routine Description:
Destroys a work queue and its thread pool.
Arguments:
WorkQueue - The queue to destroy.
Return Value:
None.
Notes:
The following rules must be observed in order to safely destroy a
work queue:
1) No new work items may be posted to the queue once all previously
posted items have been processed by this routine.
2) WorkRoutines must be able to process items until this
call returns. After the call returns, no more items will
be delivered from the specified queue.
One workable cleanup procedure is as follows: First, direct the
WorkRoutines to silently discard completed items. Next, eliminate
all sources of new work. Finally, destroy the work queue. Note that
when in discard mode, the WorkRoutines may not access any structures
which will be destroyed by eliminating the sources of new work.
--*/
{
BOOL posted;
DWORD status;
#if THREADQ_VERBOSE
ClRtlLogPrint(LOG_NOISE, "[WTQ] Destroying work queue %1!lx!\n", WorkQueue);
#endif // 0
while (WorkQueue->TotalThreads != 0) {
#if THREADQ_VERBOSE
ClRtlLogPrint(
LOG_NOISE,
"[WTQ] Destroy: Posting terminate item, thread cnt %1!u!\n",
WorkQueue->TotalThreads
);
#endif // 0
posted = PostQueuedCompletionStatus(
WorkQueue->IoCompletionPort,
0,
0,
NULL
);
if (!posted) {
status = GetLastError();
ClRtlLogPrint(
LOG_CRITICAL,
"[WTQ] Destroy: Failed to post termination item, %1!u!\n",
status
);
CL_ASSERT(status == ERROR_SUCCESS);
break;
}
#if THREADQ_VERBOSE
ClRtlLogPrint(LOG_NOISE, "[WTQ] Destroy: Waiting for a thread to terminate.\n");
#endif // 0
status = WaitForSingleObject(WorkQueue->StopEvent, INFINITE);
CL_ASSERT(status == WAIT_OBJECT_0);
#if THREADQ_VERBOSE
ClRtlLogPrint(LOG_NOISE, "[WTQ] Destroy: A thread terminated.\n");
#endif // 0
}
CloseHandle(WorkQueue->IoCompletionPort);
CloseHandle(WorkQueue->StopEvent);
LocalFree(WorkQueue);
#if THREADQ_VERBOSE
ClRtlLogPrint(LOG_NOISE, "[WTQ] Work queue %1!lx! destroyed\n", WorkQueue);
#endif // 0
return;
}
DWORD
ClRtlPostItemWorkQueue(
IN PCLRTL_WORK_QUEUE WorkQueue,
IN PCLRTL_WORK_ITEM WorkItem,
IN DWORD BytesTransferred,
IN ULONG_PTR IoContext
)
/*++
Routine Description:
Posts a specified work item to a specified work queue.
Arguments:
WorkQueue - A pointer to the work queue to which to post the item.
WorkItem - A pointer to the item to post.
BytesTransferred - If the work item represents a completed I/O operation,
this parameter contains the number of bytes
transferred during the operation. For other work items,
the semantics of this parameter may be defined by
the caller.
IoContext - If the work item represents a completed I/O operation,
this parameter contains the context value associated
with the handle on which the operation was submitted.
Of other work items, the semantics of this parameter
may be defined by the caller.
Return Value:
ERROR_SUCCESS if the item was posted successfully.
A Win32 error code if the post operation fails.
--*/
{
BOOL posted;
posted = PostQueuedCompletionStatus(
WorkQueue->IoCompletionPort,
BytesTransferred,
IoContext,
&(WorkItem->Overlapped)
);
if (posted) {
return(ERROR_SUCCESS);
}
return(GetLastError());
}
DWORD
ClRtlAssociateIoHandleWorkQueue(
IN PCLRTL_WORK_QUEUE WorkQueue,
IN HANDLE IoHandle,
IN ULONG_PTR IoContext
)
/*++
Routine Description:
Associates a specified I/O handle, opened for overlapped I/O
completion, with a work queue. All pending I/O operations on
the specified handle will be posted to the work queue when
completed. An initialized CLRTL_WORK_ITEM must be used to supply
the OVERLAPPED structure whenever an I/O operation is submitted on
the specified handle.
Arguments:
WorkQueue - The work queue with which to associate the I/O handle.
IoHandle - The I/O handle to associate.
IoContext - A context value to associate with the specified handle.
This value will be supplied as a parameter to the
WorkRoutine which processes completions for this
handle.
Return Value:
ERROR_SUCCESS if the association completes successfully.
A Win32 error code if the association fails.
--*/
{
HANDLE portHandle;
portHandle = CreateIoCompletionPort(
IoHandle,
WorkQueue->IoCompletionPort,
IoContext,
WorkQueue->ConcurrentThreads
);
if (portHandle != NULL) {
CL_ASSERT(portHandle == WorkQueue->IoCompletionPort);
return(ERROR_SUCCESS);
}
return(GetLastError());
}