// // QTCP.C version 1.0.3 // // This program tests the quality of a network connection in terms of // variation in latency (jitter). It is based on TTCP, a public domain // program, written for BSD. The version of TTCP upon which this was // based has been contributed to by: // // T.C. Slattery, USNA (18 Dec 84) // Mike Muuss and T. Slattery (16 Oct 85) // Silicon Graphics, Inc. (1989) // // QTCP is written by Yoram Bernet (yoramb@microsoft.com) // further development work by John Holmes (jsholmes@mit.edu) // // QTCP user level code may be used to provide rough jitter measurements, // which indicate both operating system and network jitter. However, QTCP // is intended to be used in conjunction with kernel timestamping for precise // jitter measurements. The kernel component timestmp.sys should be installed // when running on Win2000 (beta-3 or later). // // timestmp.sys is written by Shreedhar Madhavapeddi (shreem@microsoft.com) // // // Distribution Status - // Public Domain. Distribution Unlimited. // // Version History - // 0.8: // - adaptation of TTCP by Yoram Bernet -- core functionality // 0.9: (6/15/99) // - first version by John Holmes -- bug fixes and new features // - fixed all compile warnings // - added -v option to set up RSVP connection without sending data // - added -y option to skip confirmation of continues // - fixed line length error in log files // - fixed service type string to display properly // - added best effort and no service service types (BE & NS) // - added version string print upon execution // 0.9.1: (6/17/99) // - check for hardware clock reset using correlation coefficient // - fixed incorrect clock skew in .sta file // - fixed -v option to keep socket open until user carriage returns // - added local statistics to clock skew computation for better estimate // - added -k option to prevent using local statistics for clock skew // - fixed maximum latency computation // 0.9.2: (6/23/99) // - fixed peak rate in flowspec so no shaping happens in CL servicetype // - added -e option to force shaping // - fixed error in allocating size of log array with bufsize <= 1500 bytes // - fixed not exiting on receiver // - fixed access violation if no filename specified on receiver // - changed dummy log entries to be off by default // - added -F option to convert raw file to log file // 0.9.3: (6/29/99) // - improved low transmission speed at high packet/second rates by changing // default # async buffers from 3 to 32 // - fixed user mode timestamps to use NtQueryPerformanceCounter() // - added -u option to use usermode timestamps in log generation // 0.9.4: (7/8/99) // - cleaned up source (chopped up main into a bunch of functions to improve readability) // - fixed default buffer size to be 1472 bytes, so whole packet is 1500 bytes. // - rewrote i/o code to use callbacks for asynch i/o in order to improve throughput // - doing the right thing if not getting kernel-mode timestamps // - added ability to run for a specified amount of time with -n##s paramater // - added dynamic resizing of log array on receiver to prevent access violations // with mismatched parameters // - fixed devious bug in the GrowLogArray routine // - fixed total time reported for long runs (use UINT64 instead of DWORD) // - fixed problem with -F option specified on empty but extant file // - added RSVPMonitor Thread to watch for RSVP-err messages on receiver and // early abort by sender // - removed -a option as it is now obsolete // - revised usage screen to make more clear what pertains to sender and what // pertains to receiver // - fixed crash if receiver terminates before transmitter finishes // 0.9.5: (7/15/99) // - re-added error checking on WriteFileEx and ReadFileEx routines // 0.9.6: (7/20/99) // - changed default filler data in buffer so that it is less compressible to // better account for links that compress data before sending // - added -i option to use more compressible data // 0.9.7: (7/24/99) // - put back a thread to watch for 'q' on receiver to quit properly before sender's done // - added control channel to better handle RSVP timeouts, early aborts, etc. // - if no calibrations are specified, we calibrate based on all buffers // - gracefully exit if LogRecord runs out of memory, saving all logs we've got so far // - changed default behavior so raw file is dumped with no normalization whatsoever. // - improved the way anomalous points are caught in clock-skew calc // 0.9.8: (7/28/99) // - fixed field assignments & file opening problem on converting raw file to log. // - changed latency to be written to file to signed and fixed normalization routine for // cases when clocks are orders of magnitude different (underflow error) // - added absolute deviation as goodness of fit measure // - added routine to look for clock jumps and fix for them (with -k3 option) // 0.9.9: (8/4/99) // - changed format of .sta file to include more useful information and test params // - changed Logging scheme so that we are limited by disk space instead of memory // (now using a memory mapped file for the log, so the theoretical limit has gone from // less than 2GB to 18EB, but we won't ever get that in practice on normal disks) // - added -ni option to run indefinitely // - added -R##B option to specify tokenrate in bytes // - made default not to show dropped packets at console (it only causes more drops) // - added -q## option to log only every nth packet // 1.0.0: (8/6/99) // - fixed bug where if a new qtcp receiver is started immediately after a previous // instance, it will think "theend" packets are normal packets and AV // - added check for the piix4 timer chip and an appropriate warning // - using precise incorrect value in FixWackyTimestamps function // - added -A option (aggregate data processing of all .sta files in a directory) // 1.0.1: (8/6/99) // - fixed incorrect calculation of send rate with dropped packets // 1.0.2: (8/23/99) // - improved clock skip detection algorithm // - fixed a bug in control channel communication of TokenRate // - fixed problem with forceshape option when rate is specified in bytes // 1.0.3: (8/26/99) // - fixed SENDERS NO_SENDERS bug // - added summary over time to aggregate stats option // - changed .sta file format to include number of drops // - fixed shaping in best effort servicetype // ToDo: // - add histogram to output in .sta file and on console // - add ability to run w/o control channel connection // - mark control channel traffic as higher priority // - add more aggregate stats (time varying statistics) -- maybe fourier xform #ifndef lint static char RCSid[] = "qtcp.c $Revision: 1.0.3 $"; #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(_AMD64_) #include #endif #include "ioctl.h" #define CONTROL_PORT 7239 CHAR *VERSION_STRING = "1.0.3"; #define MAX_STRING 255 INT64 MAX_INT64=9223372036854775807; HANDLE hRSVPMonitor; DWORD idRSVPMonitor; INT64 g_BadHalAdjustment = 46869688; // this is the value on a piix4 chip SYSTEM_INFO g_si; char g_szErr[255]; CRITICAL_SECTION g_csLogRecord; EXCEPTION_RECORD g_erec; BOOL g_fOtherSideFinished = FALSE; BOOL g_fReadyForXmit = FALSE; SOCKET fd; SOCKET g_sockControl = INVALID_SOCKET; struct sockaddr_in sinhim; struct sockaddr_in sinme; short port = 5003; // UDP port number char *host; // ptr to name of host char szHisAddr[MAX_STRING]; char szMyAddr[MAX_STRING]; int trans; // 0=receive, !0=transmit mode int normalize = 0; // dump raw file after normalizing char *Name = NULL; // Name of file for logs HANDLE hRawFile = NULL; HANDLE hLogFile = NULL; HANDLE hStatFile = NULL; HANDLE hDriver = NULL; // handle to the timestmp.sys driver WSADATA WsaData; WSAEVENT QoSEvents; SYSTEMTIME systimeStart; INT64 timeStart; INT64 time0; INT64 time1; INT64 timeElapsed; CHAR* TheEnd = "TheEnd"; CHAR* ControlledLoad = "CL"; CHAR* Guaranteed = "GR"; CHAR* BestEffort = "BE"; CHAR* NoTraffic = "NT"; LPCTSTR DriverName = TEXT("\\\\.\\Timestmp"); BOOL fWackySender = FALSE; BOOL fWackyReceiver = FALSE; typedef struct { HANDLE hSocket; INT TokenRate; INT MaxSDUSize; INT BucketSize; INT MinPolicedSize; SERVICETYPE dwServiceType; CHAR *szServiceType; INT buflen; // length of buffer INT nbuf; // number of buffers to send INT64 calibration; BOOLEAN UserStamps; // By default, we use kernel mode timestamping, if available BOOLEAN SkipConfirm; // By default, we wait for user confirmation at certain times BOOLEAN RESVonly; // By default, we send data after getting the resv int SkewFitMode; // by default, 0 = no fit, 1 = chisq, 2 = chisq w/outlier removal // 3 = abs dev BOOLEAN Wait; // By default, we wait for a QoS reservation BOOLEAN Dummy; // insert dummy rows in log by default BOOLEAN PrintDrops; // report dropped packets on console BOOLEAN ForceShape; // by default, we do not force shaping on CL flows BOOLEAN RateInBytes; // KB by default BOOLEAN AggregateStats; // by default, we do not do this BOOLEAN ConvertOnly; // by default, we act normally and do not go around converting files BOOLEAN NoSenderTimestamps; BOOLEAN NoReceiverTimestamps; BOOLEAN TimedRun; // true if we're running for a specified amount of time BOOLEAN RunForever; // run until the person pushes 'q' BOOLEAN nBufUnspecified; // true if the user has not specified the -n parameter BOOLEAN RandomFiller;// by default, we use random, incompressible filler data int LoggingPeriod; // by default, 1 (log every packet) } QTCPPARAMS, *PQTCPPARAMS; QTCPPARAMS g_params; typedef struct { BOOL Done; // done if true int nWritesInProgress; // number of writes outstanding int nReadsInProgress; // number of reads outstanding int nBuffersSent; // number of buffers sent to the device int nBuffersReceived; // number of buffers received from network int nBytesTransferred; // number of bytes written to device } QTCPSTATE, *PQTCPSTATE; QTCPSTATE g_state; typedef struct { OVERLAPPED Overlapped; PVOID pBuffer; DWORD BytesWritten; } IOREQ, *PIOREQ; #define MAX_PENDING_IO_REQS 64 // number of simultaneous async calls. // This format is used for the buffer // transmitted on the wire. typedef struct _BUFFER_FORMAT{ INT64 TimeSentUser; INT64 TimeReceivedUser; INT64 TimeSent; INT64 TimeReceived; INT64 Latency; INT BufferSize; INT SequenceNumber; } BUFFER_FORMAT, *PBUFFER_FORMAT; // This format is used for the scheduling record // written based on the received buffers. typedef struct _LOG_RECORD{ INT64 TimeSentUser; INT64 TimeReceivedUser; INT64 TimeSent; INT64 TimeReceived; INT64 Latency; INT BufferSize; INT SequenceNumber; } LOG_RECORD, *PLOG_RECORD; // The LOG structure is a data abstraction for a log of LOG_RECORDS that uses memory // mapped files to have a theoretical storage limit of 18EB. It uses two buffers in memory // along with a watcher thread so that there is no delay when switching from one bit to // the next. typedef struct { INT64 nBuffersLogged; PBYTE pbMapView; // view of file in Get/SetLogElement functions INT64 qwMapViewOffset; // offset of Get/Set view in file (rounded down to allocation) char *szStorageFile; // the name of the mapped file on disk (so we can delete it) HANDLE hStorageFile; // the memory mapped file on disk HANDLE hFileMapping; // the file mapping object for our storage file INT64 qwFileSize; // the size of the storage file in bytes } LOG, *PLOG; LOG g_log; // The STATS structure keeps a record of overall statistics for the qtcp run typedef struct { char szStaFile[MAX_PATH]; char szSender[MAX_STRING]; char szReceiver[MAX_STRING]; int nBuffers; int nTokenRate; int nBytesPerBuffer; double sendrate; double recvrate; double median; double mean; double var; double kurt; double skew; double abdev; FILETIME time; int nDrops; } STATS, *PSTATS; INT64 totalBuffers; INT anomalies = 0; INT SequenceNumber = 0; INT LastSequenceNumber = -1; #define bcopy(s, d, c) memcpy((u_char *)(d), (u_char *)(s), (c)) #define bzero(d, l) memset((d), '\0', (l)) #define SENDER 1 #define RECEIVER 0 #define SECONDS_BETWEEN_HELLOS 120 // control messages #define MSGCH_DELIMITER '!' #define MSGST_RSVPERR "RSVPERR" #define MSGST_ABORT "ABORT" #define MSGST_ERROR "ERROR" #define MSGST_DONE "DONE" #define MSGST_HELLO "HELLO" #define MSGST_RATE "RATE" #define MSGST_SIZE "SIZE" #define MSGST_NUM "NUM" #define MSGST_READY "READY" #define MSGST_VER "VER" // ------------------- // Function prototypes // ------------------- VOID SetDefaults(); VOID Usage(); BOOLEAN GoodParams(); VOID SetupLogs(); VOID SetupSockets(); SOCKET OpenQoSSocket(); INT SetQoSSocket( SOCKET fd, BOOL Sending); VOID WaitForQoS( BOOL Sender, SOCKET fd); ULONG GetRsvpStatus( DWORD dwTimeout, SOCKET fd); VOID PrintRSVPStatus( ULONG code); VOID DoTransmit(); VOID WINAPI TransmitCompletionRoutine( DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED pOverlapped); VOID WINAPI DelimiterSendCompletion( DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED pOverlapped); VOID FillBuffer( CHAR *Cp, INT Cnt); INT TimeStamp( CHAR *Cp, INT Cnt); VOID DoReceive(); VOID WINAPI RecvCompletionRoutine( DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED pOverlapped); VOID LogRecord(CHAR * Buffer); BOOL CreateLog(PLOG plog, INT64 c); BOOL GetLogEntry(PLOG plog, PLOG_RECORD prec, INT64 i); BOOL DestroyLog(PLOG plog); BOOL SetLogEntry(PLOG plog, PLOG_RECORD prec, INT64 i); BOOL AddLogEntry(PLOG plog, PLOG_RECORD prec); UINT64 GetUserTime(); DWORD MyCreateFile( IN PCHAR Name, IN PCHAR Extension, OUT HANDLE *File); void AggregateStats(); int IndexOfStatRecWith(int rate, int size, INT64 time, PSTATS pStats, int cStats); BOOL GetStatsFromFile(PSTATS pstats); VOID DoStatsFromFile(); DWORD OpenRawFile( IN PCHAR Name, OUT HANDLE *File); INT64 ReadSchedulingRecords(HANDLE file); VOID DoStats(); VOID WriteSchedulingRecords( HANDLE File, BOOLEAN InsertDummyRows); void AdvancedStats(); VOID GenericStats(); VOID CheckForLostPackets(); VOID WriteStats( UCHAR * HoldingBuffer, INT Count); VOID NormalizeTimeStamps(); VOID ClockSkew( DOUBLE * Slope, DOUBLE * Offset); BOOLEAN AnomalousPoint( DOUBLE x, DOUBLE y); VOID AdjustForClockSkew( DOUBLE Slope, DOUBLE Offset); BOOL FixWackyTimestamps(); // monitor threads DWORD WINAPI RSVPMonitor (LPVOID lpvThreadParm); DWORD WINAPI KeyboardMonitor (LPVOID lpvThreadParm); DWORD WINAPI ControlSocketMonitor(LPVOID lpvThreadParm); DWORD WINAPI LogWatcher(LPVOID lpvThreadParm); // utilities int SendControlMessage(SOCKET sock, char * szMsg); void ErrorExit(char *msg, DWORD dwErrorNumber); UINT64 GetBadHalAdjustment(); //int compare( const void *arg1, const void *arg2 ); int __cdecl compare(const void *elem1, const void *elem2 ) ; void medfit(double x[], double y[], int N, double *a, double *b, double *abdev); double mode(const double data[], const int N); void RemoveDuplicates(int rg[], int * pN); void RemoveDuplicatesI64(INT64 rg[], int * pN); #define RoundUp(val, unit) (val + (val % unit)) #define InRange(val, low, high) ((val >= low) && (val < high)) ? TRUE:FALSE void PrintFlowspec(LPFLOWSPEC lpfs); VOID __cdecl main(INT argc,CHAR **argv) { int error; char *stopstring; char szBuf[MAX_STRING]; BOOL b; ULONG bytesreturned; printf("qtcp version %s\n\n",VERSION_STRING); if (GetBadHalAdjustment() == (UINT64)g_BadHalAdjustment) { printf("WARNING: This machine has a timer whose frequency matches that of the piix4\n"); printf(" chip. There is a known bug in the HAL for this timer that causes the\n"); printf(" timer to jump forward about 4.7 seconds every once in a while.\n"); printf(" If you notice large jumps in the timestamps from this machine, try\n"); printf(" running with the -k3 option to attempt to correct for the timer bug.\n\n"); } srand( (unsigned)time( NULL ) ); // seed the random number generator timeStart = GetUserTime(); GetSystemInfo(&g_si); error = WSAStartup( 0x0101, &WsaData ); if (error == SOCKET_ERROR) { printf("qtcp: WSAStartup failed %ld:", WSAGetLastError()); } if (argc < 2) Usage(); Name = malloc(MAX_STRING); bzero(Name,MAX_STRING); SetDefaults(); argv++; argc--; while( argc>0 && argv[0][0] == '-' ) { switch (argv[0][1]) { case 'B': g_params.BucketSize = atoi(&argv[0][2]); break; case 'm': g_params.MinPolicedSize = atoi(&argv[0][2]); break; case 'M': g_params.MaxSDUSize = atoi(&argv[0][2]); break; case 'R': g_params.TokenRate = (int)strtod(&argv[0][2],&stopstring); if (*stopstring == 0) { // normal run g_params.RateInBytes = FALSE; break; } if (*stopstring == 'B') { // rate is in bytes / sec, not kbytes/sec g_params.RateInBytes = TRUE; break; } else { Usage(); break; } case 'S': g_params.szServiceType = &argv[0][2]; if(!strncmp(g_params.szServiceType, ControlledLoad, 2)){ g_params.dwServiceType = SERVICETYPE_CONTROLLEDLOAD; break; } if(!strncmp(g_params.szServiceType, Guaranteed, 2)){ g_params.dwServiceType = SERVICETYPE_GUARANTEED; break; } if(!strncmp(g_params.szServiceType, BestEffort, 2)){ g_params.dwServiceType = SERVICETYPE_BESTEFFORT; g_params.Wait = FALSE; break; } if(!strncmp(g_params.szServiceType, NoTraffic, 2)){ g_params.dwServiceType = SERVICETYPE_NOTRAFFIC; g_params.Wait = FALSE; break; } fprintf(stderr, "Invalid service type (not CL or GR).\n"); fprintf(stderr, "Using GUARANTEED service.\n"); break; case 'e': g_params.ForceShape = TRUE; break; case 'W': g_params.Wait = FALSE; break; case 't': trans = 1; break; case 'f': strcpy(Name,&argv[0][2]); break; case 'F': strcpy(Name,&argv[0][2]); g_params.ConvertOnly = TRUE; break; case 'A': strcpy(Name,&argv[0][2]); g_params.AggregateStats = TRUE; break; case 'r': trans = 0; break; case 'n': g_params.nbuf = (INT)strtod(&argv[0][2],&stopstring); if (*stopstring == 0) { // normal run g_params.nBufUnspecified = FALSE; break; } if (*stopstring == 'i') { // run for an infinite time g_params.RunForever = TRUE; break; } if (*stopstring == 's') { // run for a specified time g_params.TimedRun = TRUE; printf("Running for %d seconds\n",g_params.nbuf); break; } else { Usage(); break; } case 'c': g_params.calibration = atoi(&argv[0][2]); break; case 'k': g_params.SkewFitMode = atoi(&argv[0][2]); if (g_params.SkewFitMode < 0 || g_params.SkewFitMode > 3) ErrorExit("Invalid Skew Fit Mode",g_params.SkewFitMode); break; case 'l': g_params.buflen = atoi(&argv[0][2]); break; case 'p': port = (short)atoi(&argv[0][2]); break; case 'd': g_params.Dummy = TRUE; break; case 'N': normalize = 1; break; case 'P': g_params.PrintDrops = TRUE; break; case 'v': g_params.RESVonly = TRUE; break; case 'y': g_params.SkipConfirm = TRUE; break; case 'u': g_params.UserStamps = TRUE; break; case 'i': g_params.RandomFiller = FALSE; break; case 'q': g_params.LoggingPeriod = atoi(&argv[0][2]); break; default: Usage(); } argv++; argc--; } // // Make an ioctl to Timestmp driver, if its exists about the // port to timestamp on. // printf("Trying to open %s\n", DriverName); hDriver = CreateFile(DriverName, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, // Default security OPEN_EXISTING, 0, 0); // No template if(hDriver == INVALID_HANDLE_VALUE) { printf("Timestmp.sys CreateFile- Error %ld - Maybe its not INSTALLED\n", GetLastError()); // Otherwise, print success and close the driver } else { printf("Timestmp.sys - CreateFile Success.\n"); b = DeviceIoControl( hDriver, // handle to a device, file, or directory IOCTL_TIMESTMP_REGISTER_PORT, // control code of operation to perform &port, // pointer to buffer to supply input data 2, //nInBufferSize, // size, in bytes, of input buffer NULL, //lpOutBuffer, // pointer to buffer to receive output data 0, //nOutBufferSize, // size, in bytes, of output buffer &bytesreturned, // pointer to variable to receive byte count NULL // pointer to overlapped structure ); printf("IOCTL performed\n"); if (!b) { printf("IOCTL FAILED!\n", GetLastError()); // Close the driver CloseHandle(hDriver); } else { printf("IOCTL succeeded!\n"); } } // get the host address if we're the sender if(trans) { if(argc != 1) Usage(); host = malloc(strlen(argv[0]) + 1); strcpy(host,argv[0]); } // first, we see if this is a conversion -- if it is, just jump right in, else go on if (g_params.ConvertOnly) { DoStatsFromFile(); exit(0); } // see if we're supposed to do stat aggregation if (g_params.AggregateStats) { AggregateStats(); exit(0); } // Do argument sanity tests & set default values if not already set if (!GoodParams()) exit(1); // Spawn off the control socket monitor thread CreateThread(NULL, 0, ControlSocketMonitor, (LPVOID)host, 0, NULL); // Get the sockets ready, set for QoS, and wait for a connection SetupSockets(); // Check for a RESV only session if (g_params.RESVonly) { // keep socket open and hang out fprintf(stdout, "RSVP connection established. Press return to quit.\n"); while(TRUE){ if(getchar()) break; } exit(0); } // start up the RSVPMonitor and keyboard monitor threads to watch for wackiness hRSVPMonitor = CreateThread(NULL,0,RSVPMonitor,NULL,0,&idRSVPMonitor); CreateThread(NULL,0,KeyboardMonitor,NULL,0,NULL); // wait for the control channel to be set up, if it's not already while (g_sockControl == INVALID_SOCKET) Sleep(50); if (!trans) { // we want to make sure these are not initialized, so we don't put wrong values in .sta g_params.buflen = 0; g_params.nbuf = 2048; // it's ok to init this because it's not saved in .sta g_params.TokenRate = 0; } totalBuffers = g_params.nbuf + g_params.calibration; // Tell the receiver the important parameters if (trans) { if (g_params.RunForever) totalBuffers = 2048; sprintf(szBuf, "%s %d", MSGST_NUM, totalBuffers); SendControlMessage(g_sockControl, szBuf); sprintf(szBuf, "%s %d", MSGST_SIZE, g_params.buflen); SendControlMessage(g_sockControl, szBuf); if (g_params.RateInBytes) sprintf(szBuf, "%s %d", MSGST_RATE, g_params.TokenRate); else sprintf(szBuf, "%s %d", MSGST_RATE, 1000 * g_params.TokenRate); SendControlMessage(g_sockControl, szBuf); } while (!g_fReadyForXmit) Sleep(50); // If we're the receiver, set up the log buffer and files if((Name != NULL) && !trans){ SetupLogs(); } // Let the user know what's up if(trans){ fprintf(stdout, "qtcp TRANSMITTER\n"); if (g_params.calibration) fprintf(stdout, "\tSending %d calibration buffers.\n", g_params.calibration); fprintf(stdout, "\tSending %d buffers of length %d.\n", g_params.nbuf, g_params.buflen); fprintf(stdout, "\tDestination address (port) is %s (%d).\n", argv[0], port); if (g_params.RateInBytes) fprintf(stdout, "\tToken rate is %d bytes/sec.\n", g_params.TokenRate); else fprintf(stdout, "\tToken rate is %d Kbytes/sec.\n", g_params.TokenRate); fprintf(stdout, "\tBucket size is %d bytes.\n", g_params.BucketSize); } else{ fprintf(stdout, "qtcp RECEIVER\n"); if (g_params.calibration) fprintf(stdout, "\tPrepared to receive %d calibration buffers.\n", g_params.calibration); if (!g_params.nBufUnspecified) { fprintf(stdout, "\tPrepared to receive %d buffers.\n", g_params.nbuf); } } // Do the actual communication time0 = GetUserTime(); if (trans) DoTransmit(); else DoReceive(); time1 = GetUserTime(); timeElapsed = (time1 - time0)/10000; // tell the other guy we're done SendControlMessage(g_sockControl, MSGST_DONE); // get to a new line printf("\n"); // put some stats on the transmitter console if (trans) { printf("Sent %ld bytes in %I64d milliseconds = %I64d KBps\n", g_state.nBytesTransferred, timeElapsed, g_state.nBytesTransferred/timeElapsed); } // wait for the other side to tell us it's done. while (!g_fOtherSideFinished) Sleep(50); // let the user know if timestmp.sys was installed if (g_params.NoSenderTimestamps && g_params.NoReceiverTimestamps) printf("WARNING: No kernel-level timestamps detected on sender or receiver\n\tUsing user-mode timestamps.\n"); else if (g_params.NoSenderTimestamps) printf("WARNING: No kernel-level timestamps detected on sender\n\tUsing user-mode timestamps.\n"); else if (g_params.NoReceiverTimestamps) printf("WARNING: No kernel-level timestamps detected on receiver\n Using user-mode timestamps.\n"); // Close down the sockets if (closesocket((SOCKET)g_params.hSocket) != 0) fprintf(stderr,"closesocket failed: %d\n",WSAGetLastError()); if(timeElapsed <= 100){ fprintf(stdout, "qtcp %s:Time interval too short for valid measurement!\n", trans?"-t":"-r"); } // Close files & exit if(!trans && Name != NULL){ if (g_log.nBuffersLogged) { DoStats(); } else { printf("ERROR: no buffers logged due to errors.\n"); } CloseHandle(hRawFile); CloseHandle(hLogFile); CloseHandle(hStatFile); DestroyLog(&g_log); } if (WSACleanup() != 0) fprintf(stderr,"WSACleanup failed: %d\n",WSAGetLastError()); printf("\n"); _exit(0); } // main() VOID SetDefaults() { g_params.hSocket = NULL; g_params.TokenRate = 100; g_params.MaxSDUSize = QOS_NOT_SPECIFIED; g_params.BucketSize = QOS_NOT_SPECIFIED; g_params.MinPolicedSize = QOS_NOT_SPECIFIED; g_params.dwServiceType = SERVICETYPE_GUARANTEED; g_params.szServiceType = "GR"; g_params.buflen = 1472; /* length of buffer */ g_params.nbuf = 2 * 1024; /* number of buffers to send */ g_params.calibration = 0; g_params.UserStamps = FALSE; // By default, we use kernel mode timestamping, if available g_params.SkipConfirm = FALSE; // By default, we wait for user confirmation at certain times g_params.SkewFitMode = 2; // by default, we use absolute deviation g_params.Wait = TRUE; // By default, we wait for a QoS reservation g_params.Dummy = FALSE; // insert dummy rows in log by default g_params.PrintDrops = FALSE; // report dropped packets on console g_params.ForceShape = FALSE; // by default, we do not force shaping on CL flows g_params.RateInBytes = FALSE; // KB by default g_params.ConvertOnly = FALSE; // by default, we act normally and do not go around converting files g_params.AggregateStats = FALSE; g_params.NoSenderTimestamps = FALSE; g_params.NoReceiverTimestamps = FALSE; g_params.TimedRun = FALSE; // by default, we run for a number of packets g_params.RunForever = FALSE; // by default, we run fora number of packets g_params.nBufUnspecified = TRUE; g_params.RandomFiller = TRUE; // by default, we use random filler to prevent compression g_params.LoggingPeriod = 1; } // SetDefaults() VOID Usage() { fprintf(stderr,"Usage: qtcp [-options] -t host\n"); fprintf(stderr," qtcp [-options] -r\n"); fprintf(stderr," -t options:\n"); fprintf(stderr," -B## TokenBucket size signaled to network and to traffic control\n"); fprintf(stderr," (default is equal to buffer size)\n"); fprintf(stderr," -m## MinPolicedSize signaled to network and to traffic control\n"); fprintf(stderr," (default is equal to buffer size)\n"); fprintf(stderr," -R## TokenRate in kilobytes per second (default is 100 KBPS)\n"); fprintf(stderr," -R##B TokenRate in bytes per second\n"); fprintf(stderr," -e Force shaping to TokenRate.\n"); fprintf(stderr," -M MaxSDUSize to be used in signaling messages (default is buffer\n"); fprintf(stderr," size\n"); fprintf(stderr," -l## length of buffers to transmit (default is 1472 bytes)\n"); fprintf(stderr," -n## number of source bufs written to network (default is 2048 bytes)\n"); fprintf(stderr," -n##s numbef of seconds to send buffers for (numbef of buffers will\n"); fprintf(stderr," be calculated based on other parameters\n"); fprintf(stderr," -ni run indefinitely (will stop when 'q' is pressed on either)\n"); fprintf(stderr," -c## Specifies number of calibration packets to be sent\n"); fprintf(stderr," Calibration packets will be sent immediately\n"); fprintf(stderr," After calibration packets are sent, n additional\n"); fprintf(stderr," packets will be sent. This option is useful if you want to\n"); fprintf(stderr," change network conditions after a set calibration phase\n"); fprintf(stderr," -y skip confirmation message after calibration phase\n"); fprintf(stderr," -p## port number to send to or listen at (default 5003)\n"); fprintf(stderr," -i use more compressible buffer data\n"); fprintf(stderr," -r options:\n"); fprintf(stderr," -f\"filename\" Name prefix to be used in generating log file and\n"); fprintf(stderr," statistics summary. (no file generated by default)\n"); fprintf(stderr," -c## Specifies number of buffers to use in clock-skew calibration\n"); fprintf(stderr," -k0 do not calculate clock skew\n"); fprintf(stderr," -k1 use chi squared as goodness of fit\n"); fprintf(stderr," -k2 use absolute deviation as goodness of fit (default)\n"); fprintf(stderr," -k3 use abs dev and check for clock jumps\n"); fprintf(stderr," -d suppress insertion of dummy log records for lost packets.\n"); fprintf(stderr," -N Normalize before dumping raw file (default is after)\n"); fprintf(stderr," -P enables console reporting of dropped packets\n"); fprintf(stderr," -u use user mode timestamps instead of kernel timestamps in logs\n"); fprintf(stderr," -q## log only every ##th packet\n"); fprintf(stderr," common options:\n"); fprintf(stderr," -S\"service type\" (CL or GR -- GR is default)\n"); fprintf(stderr," -W Suppress waiting for QoS reservation\n"); fprintf(stderr," -v Set up QoS reservation only, send no data\n"); fprintf(stderr," -F\"filename\" Name prefix of raw file to be converted to log file\n"); fprintf(stderr," -A\"path\" Path to directory for aggregate statistics computation\n"); WSACleanup(); exit(1); } // Usage() BOOLEAN GoodParams() { BOOLEAN ok = TRUE; if(g_params.buflen < sizeof(BUFFER_FORMAT)){ printf("Buffer size too small for record!\n"); ok = FALSE; } // Unless otherwise specified, min policed size will be equal to // buflen. if(g_params.MinPolicedSize == QOS_NOT_SPECIFIED){ g_params.MinPolicedSize = g_params.buflen; } // Same goes for bucket size if(g_params.BucketSize == QOS_NOT_SPECIFIED){ g_params.BucketSize = g_params.buflen; } // And for MaxSDU if(g_params.MaxSDUSize == QOS_NOT_SPECIFIED){ g_params.MaxSDUSize = g_params.buflen; } // If the bucket size is smaller than the buffer size, // and this is a sender, then warn the user because // data will be discarded if((g_params.BucketSize < g_params.buflen) && trans){ printf("Token bucket size is smaller than buffer size!\n"); ok = FALSE; } if(g_params.MaxSDUSize < g_params.buflen){ printf("MaxSDU cannot be less than buffer size!\n"); ok = FALSE; } if(g_params.buflen < 5){ g_params.buflen = 5; // send more than the sentinel size } if(g_params.TimedRun) { if (g_params.RateInBytes) g_params.nbuf = g_params.nbuf * g_params.TokenRate / g_params.buflen; else g_params.nbuf = g_params.nbuf * g_params.TokenRate * 1000 / g_params.buflen; printf("Using %d buffers\n",g_params.nbuf); } return ok; } // GoodParams() VOID SetupLogs() { CreateLog(&g_log, totalBuffers); // set up logging files if(ERROR_SUCCESS != MyCreateFile(Name,".raw",&hRawFile)){ fprintf(stderr, "WARNING: Could not create raw file.\n"); } if(ERROR_SUCCESS == MyCreateFile(Name,".log", &hLogFile)){ fprintf(stdout,"Logging per-packet data to %s.log.\n",Name); } else{ fprintf(stderr, "WARNING: Could not create log file.\n"); } if(ERROR_SUCCESS == MyCreateFile(Name, ".sta", &hStatFile)){ fprintf(stdout,"Writing statistics sumary to %s.sta\n",Name); } else{ fprintf(stderr,"Could not create statistics file.\n"); } } // SetupLogs() VOID SetupSockets() { struct hostent *addr; ULONG addr_tmp; char szAddr[MAX_STRING]; int dwAddrSize, dwError; // Set address and port parameters if(trans) { bzero((char *)&sinhim, sizeof(sinhim)); if (atoi(host) > 0 ) { sinhim.sin_family = AF_INET; sinhim.sin_addr.s_addr = inet_addr(host); } else{ if ((addr=gethostbyname(host)) == NULL){ printf("ERROR: bad hostname\n"); WSACleanup(); exit(1); } sinhim.sin_family = addr->h_addrtype; bcopy(addr->h_addr,(char*)&addr_tmp, addr->h_length); sinhim.sin_addr.s_addr = addr_tmp; } sinhim.sin_port = htons(port); sinme.sin_port = 0; /* free choice */ } else{ sinme.sin_port = htons(port); } sinme.sin_family = AF_INET; // Open socket for QoS traffic fd = OpenQoSSocket(); if((fd == (UINT_PTR)NULL) || (fd == INVALID_SOCKET)){ fprintf(stderr,"Failed to open QoS socket!\n"); exit(1); } // Prepare to get QoS notifications if((QoSEvents = WSACreateEvent()) == WSA_INVALID_EVENT){ fprintf(stderr, "Failed to create an event for QoS notifications %ld\n", WSAGetLastError()); exit(1); } if(WSAEventSelect(fd, QoSEvents, FD_QOS) == SOCKET_ERROR){ fprintf(stderr, "Unable to get notifications for QoS events. %ld\n", WSAGetLastError()); } if(trans){ // Set QoS on sending traffic if(SetQoSSocket(fd, TRUE)){ exit(1); } fprintf(stdout, "Initiated QoS connection. Waiting for receiver.\n"); WaitForQoS(SENDER, fd); } else{ // we're the receiver, so bind and wait if(bind(fd, (PSOCKADDR)&sinme, sizeof(sinme)) < 0){ printf("bind() failed: %ld\n", GetLastError( )); } if(SetQoSSocket(fd, FALSE)){ exit(1); } fprintf(stdout, "Waiting for QoS sender to initiate QoS connection.\n"); WaitForQoS(RECEIVER, fd); } // set some options // none to set! g_params.hSocket = (HANDLE)fd; } // SetupSockets() SOCKET OpenQoSSocket( ) { INT bufferSize = 0; INT numProtocols; LPWSAPROTOCOL_INFO installedProtocols, qosProtocol; INT i; SOCKET fd; BOOLEAN QoSInstalled = FALSE; // Call WSAEnumProtocols to determine buffer size required numProtocols = WSAEnumProtocols(NULL, NULL, &bufferSize); if((numProtocols != SOCKET_ERROR) && (WSAGetLastError() != WSAENOBUFS)){ printf("Failed to enumerate protocols!\n"); return((UINT_PTR)NULL); } else{ // Enumerate the protocols, find the QoS enabled one installedProtocols = (LPWSAPROTOCOL_INFO)malloc(bufferSize); numProtocols = WSAEnumProtocols(NULL, (LPVOID)installedProtocols, &bufferSize); if(numProtocols == SOCKET_ERROR){ printf("Failed to enumerate protocols!\n"); return((UINT_PTR)NULL); } else{ qosProtocol = installedProtocols; for(i=0; idwServiceFlags1 & XP1_QOS_SUPPORTED)&& (qosProtocol->dwServiceFlags1 & XP1_CONNECTIONLESS) && (qosProtocol->iAddressFamily == AF_INET)){ QoSInstalled = TRUE; break; } qosProtocol++; } } // Now open the socket. if (!QoSInstalled) { fprintf(stderr,"ERROR: No QoS protocols installed on this machine\n"); exit(1); } fd = WSASocket(0, SOCK_DGRAM, 0, qosProtocol, 0, WSA_FLAG_OVERLAPPED); free(installedProtocols); return(fd); } } // OpenQoSSocket() INT SetQoSSocket( SOCKET fd, BOOL Sending) { QOS qos; INT status; LPFLOWSPEC flowSpec; INT dummy; INT receiverServiceType = Sending? SERVICETYPE_NOTRAFFIC: g_params.dwServiceType; qos.ProviderSpecific.len = 0; qos.ProviderSpecific.buf = 0; // receiving flowspec is either NO_TRAFFIC (on a sender) or all // defaults except for the service type (on a receiver) flowSpec = &qos.ReceivingFlowspec; flowSpec->TokenRate = QOS_NOT_SPECIFIED; flowSpec->TokenBucketSize = QOS_NOT_SPECIFIED; flowSpec->PeakBandwidth = QOS_NOT_SPECIFIED; flowSpec->Latency = QOS_NOT_SPECIFIED; flowSpec->DelayVariation = QOS_NOT_SPECIFIED; flowSpec->ServiceType = receiverServiceType; flowSpec->MaxSduSize = QOS_NOT_SPECIFIED; flowSpec->MinimumPolicedSize = QOS_NOT_SPECIFIED; // now do the sending flowspec flowSpec = &qos.SendingFlowspec; if(Sending){ if (g_params.RateInBytes) flowSpec->TokenRate = g_params.TokenRate; else flowSpec->TokenRate = g_params.TokenRate * 1000; flowSpec->TokenBucketSize = g_params.BucketSize; if (g_params.ForceShape) { if (g_params.RateInBytes) flowSpec->PeakBandwidth = g_params.TokenRate; else flowSpec->PeakBandwidth = g_params.TokenRate * 1000; } else flowSpec->PeakBandwidth = QOS_NOT_SPECIFIED; flowSpec->Latency = QOS_NOT_SPECIFIED; flowSpec->DelayVariation = QOS_NOT_SPECIFIED; flowSpec->ServiceType = g_params.dwServiceType; if (g_params.ForceShape && flowSpec->ServiceType == SERVICETYPE_BESTEFFORT ) flowSpec->ServiceType = SERVICETYPE_GUARANTEED | SERVICE_NO_QOS_SIGNALING; flowSpec->MaxSduSize = g_params.MaxSDUSize; flowSpec->MinimumPolicedSize = g_params.MinPolicedSize; printf("Sending Flowspec\n"); PrintFlowspec(&qos.SendingFlowspec); status = WSAConnect(fd, (PSOCKADDR)&sinhim, sizeof(sinhim), NULL, NULL, &qos, NULL); if(status){ printf("SetQoS failed on socket: %ld\n", WSAGetLastError()); } } else{ flowSpec->TokenRate = QOS_NOT_SPECIFIED; flowSpec->TokenBucketSize = QOS_NOT_SPECIFIED; flowSpec->PeakBandwidth = QOS_NOT_SPECIFIED; flowSpec->Latency = QOS_NOT_SPECIFIED; flowSpec->DelayVariation = QOS_NOT_SPECIFIED; flowSpec->ServiceType = SERVICETYPE_NOTRAFFIC; flowSpec->MaxSduSize = QOS_NOT_SPECIFIED; flowSpec->MinimumPolicedSize = QOS_NOT_SPECIFIED; status = WSAIoctl(fd, SIO_SET_QOS, &qos, sizeof(QOS), NULL, 0, &dummy, NULL, NULL); if(status){ printf("SetQoS failed on socket: %ld\n", WSAGetLastError()); } } return(status); } // SetQoSSocket() VOID WaitForQoS( BOOL Sender, SOCKET fd) { ULONG status; if(!g_params.Wait){ // For best effort, we don't do anything QoS... Return // right away. In this case, the sender should be started // after the reciever, since there is no synchronization // via rsvp and data could be missed. fprintf(stdout, "WARNING: Not waiting for QoS reservation.\n"); return; } while(TRUE){ // get the statuscode, waiting for as long as it takes status = GetRsvpStatus(WSA_INFINITE,fd); switch (status) { case WSA_QOS_RECEIVERS: // at least one RESV has arrived if (Sender) fprintf(stdout, "QoS reservation installed for %s service.\n", g_params.szServiceType); break; case WSA_QOS_SENDERS: // at least one PATH has arrived if (!Sender) fprintf(stdout, "QoS sender detected using %s service.\n", g_params.szServiceType); break; default: PrintRSVPStatus(status); break; } // if we received one of the coveted status codes, break out // altogether. otherwise wait and see if we get another batch // of indications. if( ((status == WSA_QOS_RECEIVERS) && Sender) || ((status == WSA_QOS_SENDERS) && !Sender) ) { break; } } } // WaitForQoS() ULONG GetRsvpStatus( DWORD dwTimeout, SOCKET fd) { LPQOS qos; UCHAR qosBuffer[500]; LPRSVP_STATUS_INFO rsvpStatus; INT bytesReturned; qos = (LPQOS)qosBuffer; qos->ProviderSpecific.len = sizeof(RSVP_STATUS_INFO); qos->ProviderSpecific.buf = (PUCHAR)(qos+1); // wait for notification that a QoS event has occured WSAWaitForMultipleEvents(1, &QoSEvents, FALSE, dwTimeout, TRUE); // loop through all qos events WSAIoctl(fd, SIO_GET_QOS, NULL, 0, qosBuffer, sizeof(qosBuffer), &bytesReturned, NULL, NULL); rsvpStatus = (LPRSVP_STATUS_INFO)qos->ProviderSpecific.buf; return rsvpStatus->StatusCode; } // GetRsvpStatus VOID PrintRSVPStatus(ULONG code) { switch (code) { case WSA_QOS_RECEIVERS: // at least one RESV has arrived printf("WSA_QOS_RECEIVERS\n"); break; case WSA_QOS_SENDERS: // at least one PATH has arrived printf("WSA_QOS_SENDERS\n"); break; case WSA_QOS_REQUEST_CONFIRMED: // Reserve has been confirmed printf("WSA_QOS_REQUEST_CONFIRMED\n"); break; case WSA_QOS_ADMISSION_FAILURE: // error due to lack of resources printf("WSA_QOS_ADMISSION_FAILURE\n"); break; case WSA_QOS_POLICY_FAILURE: // rejected for admin reasons printf("WSA_QOS_POLICY_FAILURE\n"); break; case WSA_QOS_BAD_STYLE: // unknown or conflicting style printf("WSA_QOS_BAD_STYLE\n"); break; case WSA_QOS_BAD_OBJECT: // problem with some part of the // filterspec/providerspecific // buffer in general printf("WSA_QOS_BAD_OBJECT\n"); break; case WSA_QOS_TRAFFIC_CTRL_ERROR: // problem with some part of the // flowspec printf("WSA_QOS_TRAFFIC_CTRL_ERROR\n"); break; case WSA_QOS_GENERIC_ERROR: // general error printf("WSA_QOS_GENERIC_ERROR\n"); break; default: printf("Unknown RSVP StatusCode %lu\n", code); break; } } // PrintRSVPStatus VOID DoTransmit() { IOREQ IOReq[MAX_PENDING_IO_REQS] = { 0 }; INT i; BOOL ret; BOOL fOk; g_state.nBytesTransferred = 0; g_state.nBuffersSent = 0; g_state.nWritesInProgress = 0; // fill up the initial buffers and send them on their way for (i=0; i 0) { SleepEx(INFINITE, TRUE); } // send the end of transmission delimiters for (i=0; i 0) { SleepEx(INFINITE, TRUE); } // free up the used memory for (i=0; ipBuffer,g_params.buflen); TimeStamp(pIOReq->pBuffer,g_params.buflen); // send a request to write the new buffer fOk = WriteFileEx(g_params.hSocket, pIOReq->pBuffer, g_params.buflen, pOverlapped, TransmitCompletionRoutine); if (!fOk) { printf("WriteFileEx() failed: %lu\n",GetLastError()); } g_state.nWritesInProgress++; g_state.nBuffersSent++; } } // TransmitCompletionRoutine() VOID WINAPI DelimiterSendCompletion( DWORD dwErrorCode, DWORD dwNumberOfBytesTransferred, LPOVERLAPPED pOverlapped) { g_state.nWritesInProgress--; } // DelimiterSendCompletion() VOID FillBuffer( CHAR *Cp, INT Cnt) { PBUFFER_FORMAT buf = (PBUFFER_FORMAT) Cp; CHAR c = 0; // Fill with a background pattern if (g_params.RandomFiller) { // incompressible while(Cnt-- > 0) { c = rand() % 0x5F; c += 0x20; *Cp++ = c; } } else { // compressible while(Cnt-- > 0){ while(!isprint((c&0x7F))) c++; *Cp++ = (c++&0x7F); } } buf->TimeSent = -1; buf->TimeReceived = -1; } // FillBuffer() INT TimeStamp( CHAR *Cp, INT Cnt) { PBUFFER_FORMAT record; LARGE_INTEGER timeSent; INT64 time; record = (BUFFER_FORMAT *)Cp; // Stamp with length and sequence number if(Cnt < sizeof(BUFFER_FORMAT)){ printf("ERROR: Buffer length smaller than record size!\n"); return(0); } else{ time = GetUserTime(); record->TimeSentUser = time; record->BufferSize = Cnt; record->SequenceNumber = SequenceNumber++; } return 1; } // TimeStamp() VOID DoReceive() { IOREQ IOReq[MAX_PENDING_IO_REQS] = { 0 }; INT i; BOOL ret; // set the start state g_state.Done = FALSE; g_state.nBytesTransferred = 0; g_state.nBuffersReceived = 0; g_state.nReadsInProgress = 0; // fill up the initial buffers and send them on their way for (i=0; i 0) && !g_state.Done) { SleepEx(5000, TRUE); if (g_state.Done) break; } DeleteCriticalSection(&g_csLogRecord); // cancel the other pending reads CancelIo(g_params.hSocket); // free up the used memory for (i=0; ipBuffer, TheEnd, 6))) { totalBuffers = g_state.nBuffersReceived; g_state.Done = TRUE; } // check to see if someone's set our done flag (if they have, leave) if (g_state.Done) return; // if not, then the buffer should hold a scheduling record. if(dwNumberOfBytesTransferred>0 && dwNumberOfBytesTransferred <= sizeof(BUFFER_FORMAT)) { printf("Buffer too small for scheduling record\n"); printf("\tOnly %d bytes read.\n", dwNumberOfBytesTransferred); } // Log the record, but don't log more than one at a time (lock on this call) if (dwNumberOfBytesTransferred >= sizeof(BUFFER_FORMAT) && g_state.nBuffersReceived % g_params.LoggingPeriod == 0) { EnterCriticalSection(&g_csLogRecord); LogRecord(pIOReq->pBuffer); LeaveCriticalSection(&g_csLogRecord); } // if there are more buffers (or if we don't know how many are coming), ask for one if ((g_state.nBuffersReceived < totalBuffers) || g_params.nBufUnspecified) { // send a request to read the next buffer fOk = ReadFileEx(g_params.hSocket, pIOReq->pBuffer, g_params.buflen, pOverlapped, RecvCompletionRoutine); if (!fOk) { printf("ReadFileEx() failed: %lu\n",GetLastError()); } g_state.nReadsInProgress++; g_state.nBuffersReceived++; } } // RecvCompletionRoutine() void LogRecord(char * Buffer) { // This function copies the recieved record to the scheduling array. // The contents of the array are processed and written to file once // reception is complete. PBUFFER_FORMAT inRecord = (PBUFFER_FORMAT)Buffer; LOG_RECORD outRecord; INT64 time; SYSTEMTIME CurrentTime; time = GetUserTime(); outRecord.TimeSentUser = inRecord->TimeSentUser; outRecord.TimeReceivedUser = time; outRecord.TimeSent = inRecord->TimeSent; outRecord.TimeReceived = inRecord->TimeReceived; outRecord.BufferSize = inRecord->BufferSize; outRecord.SequenceNumber = inRecord->SequenceNumber; if (inRecord->TimeSent == -1) { outRecord.TimeSent = outRecord.TimeSentUser; g_params.NoSenderTimestamps = TRUE; } if (inRecord->TimeReceived == -1) { outRecord.TimeReceived = outRecord.TimeReceivedUser; g_params.NoReceiverTimestamps = TRUE; } if(g_params.UserStamps){ outRecord.TimeSent = outRecord.TimeSentUser; outRecord.TimeReceived = outRecord.TimeReceivedUser; } outRecord.Latency = outRecord.TimeReceived - outRecord.TimeSent; AddLogEntry(&g_log, &outRecord); if(g_params.PrintDrops){ if(inRecord->SequenceNumber != LastSequenceNumber + g_params.LoggingPeriod){ GetLocalTime(&CurrentTime); printf("\n%4d/%02d/%02d %02d:%02d:%02d:%04d: ", CurrentTime.wYear, CurrentTime.wMonth, CurrentTime.wDay, CurrentTime.wHour, CurrentTime.wMinute, CurrentTime.wSecond, CurrentTime.wMilliseconds); printf("Dropped %d packets after packet %d.\n", inRecord->SequenceNumber - LastSequenceNumber, LastSequenceNumber); } LastSequenceNumber = inRecord->SequenceNumber; } return; } // LogRecord() BOOL CreateLog(PLOG plog, INT64 c) { // sets up a log structure that can hold c entries char szTempFile[MAX_PATH]; char szTempPath[MAX_PATH]; SYSTEM_INFO si; DWORD dwFileSizeHigh; DWORD dwFileSizeLow; INT64 qwFileSize; // get some system info GetSystemInfo(&si); // allocate logging array plog->nBuffersLogged = 0; plog->pbMapView = NULL; plog->qwMapViewOffset = -1; // set up the temporary storage file for logging GetTempPath(MAX_PATH, szTempPath); GetTempFileName(szTempPath, "qtc", 0, szTempFile); plog->szStorageFile = malloc(strlen(szTempFile) + 1); strcpy(plog->szStorageFile, szTempFile); plog->hStorageFile = CreateFile(szTempFile, GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY, NULL); if (plog->hStorageFile == INVALID_HANDLE_VALUE) ErrorExit("Could not create temp storage file",GetLastError()); // create the memory mapping kernel object qwFileSize = c * sizeof(LOG_RECORD); dwFileSizeHigh = (DWORD) (qwFileSize >> 32); dwFileSizeLow = (DWORD) (qwFileSize & 0xFFFFFFFF); plog->qwFileSize = qwFileSize; plog->hFileMapping = CreateFileMapping(plog->hStorageFile, NULL, PAGE_READWRITE, dwFileSizeHigh,dwFileSizeLow,NULL); if (plog->hFileMapping == NULL) ErrorExit("Could not create mapping for temp storage file",GetLastError()); return TRUE; } BOOL DestroyLog(PLOG plog) { DWORD dwError; // destroys the log and all associated data dwError = CloseHandle(plog->hFileMapping); if (!dwError) printf("Error in DestroyLog:CloseHandle(FileMapping) %d\n",GetLastError()); dwError = CloseHandle(plog->hStorageFile); if (!dwError) printf("Error in DestroyLog:CloseHandle(StorageFile) %d\n",GetLastError()); dwError = UnmapViewOfFile(plog->pbMapView); if (!dwError) printf("Error in DestroyLog:UnmapViewOfFile(plog->pbMapView) %d\n",GetLastError()); dwError = DeleteFile(plog->szStorageFile); if (!dwError) printf("Error in DestroyLog:DeleteFile(StroageFile) %d\n",GetLastError()); free(plog->szStorageFile); return FALSE; } void PrintLogRecord(PLOG_RECORD prec) { char szBuf[MAX_STRING]; sprintf(szBuf,"%d: %I64u - %I64u (%I64d)", prec->SequenceNumber,prec->TimeSent,prec->TimeReceived,prec->Latency); puts(szBuf); } BOOL ExtendLog(PLOG plog) { // makes the log bigger by some fixed constant HANDLE hNewFileMapping; INT64 qwNewFileSize; UnmapViewOfFile(plog->pbMapView); qwNewFileSize = plog->qwFileSize + g_si.dwAllocationGranularity * sizeof(LOG_RECORD); hNewFileMapping = CreateFileMapping(plog->hStorageFile, NULL, PAGE_READWRITE, (DWORD)(qwNewFileSize >> 32), (DWORD)(qwNewFileSize & 0xFFFFFFFF), NULL); if (hNewFileMapping == NULL) { ErrorExit("Could not create mapping for temp storage file",GetLastError()); return FALSE; } plog->qwFileSize = qwNewFileSize; CloseHandle(plog->hFileMapping); plog->hFileMapping = hNewFileMapping; plog->qwMapViewOffset = -1; return TRUE; } BOOL GetLogEntry(PLOG plog, PLOG_RECORD prec, INT64 i) { // fills prec with the (0 indexed) i'th log in plog // returns TRUE if it was successful, FALSE otherwise INT64 qwT; PLOG_RECORD entry; // first, check to see if this is within the range of our file if ((INT64)((i+1)*sizeof(LOG_RECORD)) > plog->qwFileSize) { // too high, so we return false return FALSE; } // we have to round down to the nearest allocation boundary qwT = sizeof(LOG_RECORD) * i; // offset within file qwT /= g_si.dwAllocationGranularity; // in allocation granularity units // check to see if we do not already have this mapped in memory if (plog->qwMapViewOffset != qwT * g_si.dwAllocationGranularity) { if (plog->pbMapView != NULL) UnmapViewOfFile(plog->pbMapView); plog->qwMapViewOffset = qwT * g_si.dwAllocationGranularity; // offset of lower allocation bound if (plog->qwFileSize < (INT64)g_si.dwAllocationGranularity) { // file is smaller than allocation granularity plog->qwMapViewOffset = 0; plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, 0, 0, 0); } else if (plog->qwFileSize - plog->qwMapViewOffset < g_si.dwAllocationGranularity) { // we're within an allocation granularity of the end of the file plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, (DWORD)(plog->qwMapViewOffset >> 32), (DWORD)(plog->qwMapViewOffset & 0xFFFFFFFF), (DWORD)(plog->qwFileSize - plog->qwMapViewOffset)); } else { // we're just somewhere in the file with space around us plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, (DWORD)(plog->qwMapViewOffset >> 32), (DWORD)(plog->qwMapViewOffset & 0xFFFFFFFF), RoundUp(g_si.dwAllocationGranularity,sizeof(LOG_RECORD))); } if (plog->pbMapView == NULL) ErrorExit("GetLogEntry could not MapViewOfFile",GetLastError()); } qwT = sizeof(LOG_RECORD) * i; entry = (PLOG_RECORD)(plog->pbMapView + (qwT - plog->qwMapViewOffset)); CopyMemory(prec, entry, sizeof(LOG_RECORD)); return TRUE; } BOOL SetLogEntry(PLOG plog, PLOG_RECORD prec, INT64 i) { // fills log entry i with the data pointed to by prec // returns TRUE if it was successful, FALSE otherwise INT64 qwT; PLOG_RECORD entry; // first, check to see if this is within the range of our file if ((INT64)((i+1)*sizeof(LOG_RECORD)) > plog->qwFileSize) { // we need to make our mapping bigger ExtendLog(plog); } // we have to round down to the nearest allocation boundary qwT = sizeof(LOG_RECORD) * i; // offset within file qwT /= g_si.dwAllocationGranularity; // in allocation granularity units // check to see if we do not already have this mapped in memory if (plog->qwMapViewOffset != qwT * g_si.dwAllocationGranularity) { if (plog->pbMapView != NULL) UnmapViewOfFile(plog->pbMapView); plog->qwMapViewOffset = qwT * g_si.dwAllocationGranularity; // offset of lower allocation bound if (plog->qwFileSize < (INT64)g_si.dwAllocationGranularity) { // file is smaller than allocation granularity plog->qwMapViewOffset = 0; plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, 0, 0, 0); } else if (plog->qwFileSize - plog->qwMapViewOffset < g_si.dwAllocationGranularity) { // we're within an allocation granularity of the end of the file plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, (DWORD)(plog->qwMapViewOffset >> 32), (DWORD)(plog->qwMapViewOffset & 0xFFFFFFFF), (DWORD)(plog->qwFileSize - plog->qwMapViewOffset)); } else { // we're just somewhere in the file with space around us plog->pbMapView = MapViewOfFile(plog->hFileMapping, FILE_MAP_WRITE, (DWORD)(plog->qwMapViewOffset >> 32), (DWORD)(plog->qwMapViewOffset & 0xFFFFFFFF), RoundUp(g_si.dwAllocationGranularity,sizeof(LOG_RECORD))); } if (plog->pbMapView == NULL) ErrorExit("SetLogEntry could not MapViewOfFile",GetLastError()); } qwT = sizeof(LOG_RECORD) * i; entry = (PLOG_RECORD)(plog->pbMapView + (qwT - plog->qwMapViewOffset)); CopyMemory(entry, prec, sizeof(LOG_RECORD)); return TRUE; } BOOL AddLogEntry(PLOG plog, PLOG_RECORD prec) { PLOG_RECORD entry; // adds the data pointed to by prec to the end of the log // returns TRUE if it was successful, FALSE otherwise SetLogEntry(plog, prec, plog->nBuffersLogged); plog->nBuffersLogged++; return TRUE; } UINT64 GetUserTime() { // This function returns the performance counter time in units of 100ns LARGE_INTEGER count, freq; NtQueryPerformanceCounter(&count,&freq); // make sure we have hardware performance counting if(freq.QuadPart == 0) { NtQuerySystemTime(&count); return (UINT64)count.QuadPart; } return (UINT64)((10000000 * count.QuadPart) / freq.QuadPart); } // GetUserTime() UINT64 GetBadHalAdjustment() { // this function returns the amount the hal timer in a machine with // an intel chipset with the piix4 timer chip will jump forward in the case of // repeated garbage returned fom the piix4 (bug #347410) so we can correct it out // in the FixWackyTimestamps routine LARGE_INTEGER freq; UINT64 diff; QueryPerformanceFrequency(&freq); // so we want to find how much it is increased in 100ns intervals if we increase // byte 3 by 1. diff = 0x01000000; diff *= 10000000; diff /= (UINT64)freq.QuadPart; return diff; } DWORD MyCreateFile( IN PCHAR Name, IN PCHAR Extension, OUT HANDLE *File) { HANDLE hFile; UCHAR * fileName; fileName = malloc(strlen(Name) + 5); bzero(fileName,strlen(Name) + 5); strncpy(fileName, Name, strlen(Name)); if (strlen(Extension)==4) { strcat(fileName,Extension); } else return !ERROR_SUCCESS; hFile = CreateFile(fileName, GENERIC_WRITE | GENERIC_READ, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); *File = hFile; return(INVALID_HANDLE_VALUE == hFile ? (!(ERROR_SUCCESS)) : ERROR_SUCCESS); } // MyCreateFile() void AggregateStats() { // this will go through the directory specified in Name and aggregate stats from // all the .sta files therein. it will then output the results of the aggregation // in a file within that directory called stats.qtc char szDirPath[3 * MAX_PATH]; char szSearchString[3 * MAX_PATH]; WIN32_FIND_DATA FileData; // Data structure describes the file found HANDLE hSearch; // Search handle returned by FindFirstFile PCHAR rgszStaFiles[1000]; // an array of the names of the .sta files int cStaFiles = 0, i,j,k,l; // keeps track of how many of the .sta files there are STATS * pStats; int rgSizes[1000], cSizes = 0; int rgRates[1000], cRates = 0; char szAggFile[3 * MAX_PATH]; char szLineBuf[1000]; STATS statsT; FILE *pfile; FILETIME rgtime[1000]; SYSTEMTIME st; ULARGE_INTEGER uliT; int ctime = 0; int cSpecs = 0; PathCanonicalize(szDirPath,Name); if (szDirPath[strlen(szDirPath) - 1] == '"') szDirPath[strlen(szDirPath) - 1] = 0; if (!PathIsDirectory(szDirPath)) { printf("Path (%s) is not a directory\n",szDirPath); ErrorExit("Invalid Path for aggregate stats", -1); } // so now szDirPath is the path to the directory we want to go through // and we begin our search for .sta files sprintf(szSearchString,"%s\\*.sta",szDirPath); hSearch = FindFirstFile (szSearchString, &FileData); if (hSearch == INVALID_HANDLE_VALUE) { ErrorExit("No .sta files found.",GetLastError()); } do { rgszStaFiles[cStaFiles] = malloc(sizeof(char) * 3 * MAX_PATH); // check to see if it's a good .sta file sprintf(statsT.szStaFile,"%s\\%s", szDirPath, FileData.cFileName); if (GetStatsFromFile(&statsT)) { // if it's good, include it strcpy(rgszStaFiles[cStaFiles], FileData.cFileName); cStaFiles++; } } while (FindNextFile(hSearch, &FileData)); if (GetLastError() != ERROR_NO_MORE_FILES) { ErrorExit("Problem in FindNextFile()",GetLastError()); } // open the stats file sprintf(szAggFile,"%s\\stats.qtc",szDirPath); pfile = fopen(szAggFile,"w+"); if (pfile == NULL) printf("Could not open file for aggregate stats: %s\n",szAggFile); pStats = malloc(cStaFiles * sizeof(STATS)); ZeroMemory(pStats, cStaFiles * sizeof(STATS)); for (i=0; i 0) { FileTimeToSystemTime(&pStats[l].time, &st); fprintf(pfile,"%02hu/%02hu/%04hu %2hu:%02hu.%02hu.%03hu: %10.1lf %10.1lf %10.1lf %8.2lf %8.2lf %10.1lf %10.1lf %10d %10d\n", st.wMonth, st.wDay, st.wYear, st.wHour, st.wMinute, st.wSecond, st.wMilliseconds, pStats[l].median, sqrt((double)pStats[l].var), pStats[l].mean, pStats[l].skew, pStats[l].kurt, pStats[l].sendrate, pStats[l].recvrate, pStats[l].nBuffers, pStats[l].nDrops); } } fprintf(pfile,"\n"); } } } fprintf(pfile, "Latency Characteristics by flowspec\n"); // --- do the stats by flowspec --- // now write the file, line by line, to szLineBuf, then to the file // median fprintf(pfile,"Median Latency (microseconds)\n"); fprintf(pfile," "); for (i=0; iszStaFile,GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); dwFileSize = GetFileSize(hFile, NULL); if (dwFileSize == 0) return FALSE; // read the whole file into szBuf ReadFile(hFile, szBuf, dwFileSize, &dwRead, NULL); // close the file CloseHandle(hFile); // parse the buffer nFields = sscanf(szBuf, "Sender: %s Receiver: %s\n" \ "First packet received: %hu:%hu.%hu.%hu %hu/%hu/%hu (UTC)\n" \ "Buffer size: %d\tTokenrate: %d\n" \ "Received %d packets.\n" \ "Logged %d records.\n" \ "Received %d bytes in %d milliseconds = %d KBps\n" \ "Clock skew is %lf microseconds per second.\n " \ "\tbased on %d calibration points\n" \ "Overall send rate: %lf Bytes/s\n" \ "Overall recv rate: %lf Bytes/s\n" \ "Latency Statistics (microsecond units): median: %lf\n" \ "\tMean: %lf\tStdev: %lf\tAbDev: %lf\n" \ "\tVariance: %lf\tSkew: %lf\t Kurtosis: %lf \n" \ "Dropped %d packets\n", pstats->szSender, pstats->szReceiver, &(st.wHour), &(st.wMinute), &(st.wSecond), &(st.wMilliseconds), &(st.wDay), &(st.wMonth), &(st.wYear), &(pstats->nBytesPerBuffer), &(pstats->nTokenRate), &(pstats->nBuffers), &nT2, &nT3, &nT4, &nT5, &T1, &nT6, &(pstats->sendrate), &(pstats->recvrate), &(pstats->median), &(pstats->mean),&T2,&(pstats->abdev), &(pstats->var),&(pstats->skew),&(pstats->kurt), &(pstats->nDrops)); if (nFields != 28 && nFields != 27) { // see if they ran without clock skew calc nFields = sscanf(szBuf, "Sender: %s Receiver: %s\n" \ "First packet received: %hu:%hu.%hu.%hu %hu/%hu/%hu (UTC)\n" \ "Buffer size: %d\tTokenrate: %d\n" \ "Received %d packets.\n" \ "Logged %d records.\n" \ "Received %d bytes in %d milliseconds = %d KBps\n" \ "Overall send rate: %lf Bytes/s\n" \ "Overall recv rate: %lf Bytes/s\n" \ "Latency Statistics (microsecond units): median: %lf\n" \ "\tMean: %lf\tStdev: %lf\tAbDev: %lf\n" \ "\tVariance: %lf\tSkew: %lf\t Kurtosis: %lf \n" \ "Dropped %d packets\n", pstats->szSender, pstats->szReceiver, &(st.wHour), &(st.wMinute), &(st.wSecond), &(st.wMilliseconds), &(st.wDay), &(st.wMonth), &(st.wYear), &(pstats->nBytesPerBuffer), &(pstats->nTokenRate), &nT1, &nT2, &nT3, &nT4, &nT5, &(pstats->sendrate), &(pstats->recvrate), &(pstats->median), &(pstats->mean),&T2,&(pstats->abdev), &(pstats->var),&(pstats->skew),&(pstats->kurt), &(pstats->nDrops)); if (nFields != 26 && nFields != 25) return FALSE; } // assemble a FILETIME structure from the date & time if (!SystemTimeToFileTime(&st,&pstats->time)) { return FALSE; } free(szBuf); return TRUE; } VOID DoStatsFromFile() { DOUBLE slope = 0; DOUBLE offset = 0; printf("Logging stats from file.\n"); if (Name == NULL) { fprintf(stderr, "ERROR: you must specify a file to convert\n"); } if(MyCreateFile(Name, ".log", &hLogFile) != ERROR_SUCCESS) { fprintf(stderr, "ERROR: could not create log file\n"); exit(1); } if(OpenRawFile(Name, &hRawFile) != ERROR_SUCCESS) { fprintf(stderr, "ERROR: could not open raw file\n"); exit(1); } ReadSchedulingRecords(hRawFile); if (g_params.calibration == 0) g_params.calibration = g_log.nBuffersLogged; NormalizeTimeStamps(); // here we check for wacky timestamps on the sender and receiver if (g_params.SkewFitMode == 3) FixWackyTimestamps(); if (g_params.SkewFitMode) { ClockSkew(&slope, &offset); AdjustForClockSkew(slope,offset); NormalizeTimeStamps(); } if(hLogFile != INVALID_HANDLE_VALUE) { WriteSchedulingRecords(hLogFile, g_params.Dummy); } printf("Done stats from file.\n"); } // DoStatsFromFile() DWORD OpenRawFile( IN PCHAR Name, OUT HANDLE *File ) { HANDLE hFile; UCHAR * logName; logName = malloc(strlen(Name) + 4); strncpy(logName, Name, strlen(Name)); logName[strlen(Name)+0] = '.'; logName[strlen(Name)+1] = 'r'; logName[strlen(Name)+2] = 'a'; logName[strlen(Name)+3] = 'w'; logName[strlen(Name)+4] = (UCHAR)NULL; hFile = CreateFile(logName, GENERIC_READ, 0, NULL, OPEN_EXISTING , FILE_ATTRIBUTE_NORMAL, NULL); *File = hFile; return(INVALID_HANDLE_VALUE == hFile ? (!(ERROR_SUCCESS)) : ERROR_SUCCESS); } // OpenRawFile() INT64 ReadSchedulingRecords(HANDLE File) { char szTempFile[MAX_PATH]; char szTempPath[MAX_PATH]; LOG_RECORD currentRecord; CHAR lineBuf[MAX_STRING]; CHAR nextChar[2] = {0,0}; DWORD readBytes = 0; INT assignedFields; if (!File || (File == INVALID_HANDLE_VALUE)) { fprintf(stderr,"ERROR: Invalid File\n"); return 0; } CreateLog(&g_log, 2048); // loop through the file, reading in line after line do { // get the next line of characters bzero(lineBuf, MAX_STRING); ZeroMemory(lineBuf, MAX_STRING); do { ReadFile(File,nextChar,1,&readBytes,NULL); if (readBytes == 0) { if (g_log.nBuffersLogged == 0) { fprintf(stderr,"ERROR: no logs read\n"); exit(1); } break; } strcat(lineBuf,nextChar); } while (*nextChar != '\n'); // parse line and add it to the log assignedFields = sscanf(lineBuf, "%I64u:%I64u:%I64u:%d:%d\n", &(currentRecord.TimeSent), &(currentRecord.TimeReceived), &(currentRecord.Latency), &(currentRecord.BufferSize), &(currentRecord.SequenceNumber)); if ((assignedFields != 5) && (assignedFields != EOF)) printf("ERROR: parsing the log gave bad field assignments on record %d\n", g_log.nBuffersLogged); if (assignedFields == EOF) break; AddLogEntry(&g_log, ¤tRecord); } while (readBytes != 0); printf("read %d records\n",g_log.nBuffersLogged); return g_log.nBuffersLogged; // return the number of records read } // ReadSchedulingRecords() VOID DoStats() { DOUBLE slope = 0; DOUBLE offset = 0; GenericStats(); if(!normalize){ if(hRawFile != INVALID_HANDLE_VALUE){ WriteSchedulingRecords(hRawFile, FALSE); } } NormalizeTimeStamps(); if(normalize){ if(hRawFile != INVALID_HANDLE_VALUE){ WriteSchedulingRecords(hRawFile, FALSE); } } if(!g_params.calibration) { // if we have nothing specified, calibrate on all buffers g_params.calibration = g_state.nBuffersReceived; } // here we check for wacky timestamps on the sender and receiver if (g_params.SkewFitMode == 3) FixWackyTimestamps(); if(g_params.SkewFitMode) { ClockSkew(&slope, &offset); AdjustForClockSkew(slope, offset); NormalizeTimeStamps(); } // we calculate these stats on the normalized / skew adjusted data AdvancedStats(); CheckForLostPackets(); if(hLogFile != INVALID_HANDLE_VALUE){ WriteSchedulingRecords(hLogFile, g_params.Dummy); } printf("\n"); } VOID WriteSchedulingRecords( HANDLE File, BOOLEAN InsertDummyRows) { LOG_RECORD scheduleRecord; CHAR formattingBuffer[MAX_STRING]; INT dwWritten; INT64 records = g_log.nBuffersLogged; INT wrote; INT i; INT64 maxLatency = (INT64)0; if(!File || (File == INVALID_HANDLE_VALUE)){ return; } while(records){ GetLogEntry(&g_log, &scheduleRecord, g_log.nBuffersLogged - records); ZeroMemory(formattingBuffer,MAX_STRING); wrote = sprintf(formattingBuffer, "%020I64u:%020I64u:%010I64d:%10d:%10d\n", scheduleRecord.TimeSent, scheduleRecord.TimeReceived, scheduleRecord.Latency, scheduleRecord.BufferSize, scheduleRecord.SequenceNumber); WriteFile(File, formattingBuffer, wrote, &dwWritten, NULL); records--; } } // WriteSchedulingRecords() VOID GenericStats() { INT bytesWritten; UCHAR holdingBuffer[MAX_STRING]; INT count; // say who the sender and receiver are count = sprintf(holdingBuffer, "Sender: %s Receiver: %s\n",szHisAddr, szMyAddr); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); // say when we received the first packet count = sprintf(holdingBuffer, "First packet received: %02u:%02u.%02u.%03u %02u/%02u/%04u (UTC)\n", systimeStart.wHour, systimeStart.wMinute, systimeStart.wSecond, systimeStart.wMilliseconds, systimeStart.wDay, systimeStart.wMonth, systimeStart.wYear); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); // write the test params to the .sta file bzero(holdingBuffer, MAX_STRING); count = _snprintf(holdingBuffer, MAX_STRING -1, "Buffer size: %d\tTokenrate: %d\n", g_params.buflen, g_params.TokenRate); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); // write some generic results bzero(holdingBuffer, MAX_STRING); count = _snprintf(holdingBuffer, MAX_STRING-1, // leave room for NULL "Received %u packets.\n", g_state.nBuffersReceived); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); bzero(holdingBuffer, MAX_STRING); count = _snprintf(holdingBuffer, MAX_STRING-1, // leave room for NULL "Logged %I64u records.\n", g_log.nBuffersLogged); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); bzero(holdingBuffer, MAX_STRING); count = _snprintf(holdingBuffer, MAX_STRING-1, // room for NULL "Received %ld bytes in %I64d milliseconds = %I64d KBps\n", g_state.nBytesTransferred, timeElapsed, g_state.nBytesTransferred/timeElapsed); WriteStats(holdingBuffer, count); printf("%s",holdingBuffer); } // GenericStats() void AdvancedStats() { // write some more interesting stats to the .sta file char szBuf[MAX_STRING]; INT64 i,n; int count; INT64 FirstTime,LastTime; double rate, median, mean, var, abdev, skew, kurt, sdev, ep = 0.0, s, p; LOG_RECORD rec; double * sortedLatencies; // overall send rate GetLogEntry(&g_log, &rec, 0); FirstTime = rec.TimeSent; GetLogEntry(&g_log, &rec, g_log.nBuffersLogged - 1); LastTime = rec.TimeSent; rate = (rec.SequenceNumber * g_params.buflen)/((double)(LastTime - FirstTime)/10000000.0); count = sprintf(szBuf, "Overall send rate: %.3f Bytes/s\n",rate); WriteStats(szBuf, count); printf("%s",szBuf); GetLogEntry(&g_log, &rec, 0); FirstTime = rec.TimeReceived; GetLogEntry(&g_log, &rec, g_log.nBuffersLogged - 1); LastTime = rec.TimeReceived; rate = (g_state.nBytesTransferred)/((double)(LastTime - FirstTime)/10000000.0); count = sprintf(szBuf, "Overall recv rate: %.3f Bytes/s\n",rate); WriteStats(szBuf, count); printf("%s",szBuf); // now show mean, variance, avdev, etc of latency. s = 0.0; n = g_log.nBuffersLogged; sortedLatencies = malloc(sizeof(double) * (UINT)n); for (i=0; i < n; i++) { // first pass, we get mean GetLogEntry(&g_log, &rec, i); s += (double)rec.Latency/10.0; sortedLatencies[i] = (double)rec.Latency/10.0; } qsort(sortedLatencies,(UINT)n,sizeof(double),compare); median = (n & 1) ? sortedLatencies[(n-1)/2] : 0.5*(sortedLatencies[n/2] + sortedLatencies[n/2 - 1]); free(sortedLatencies); mean = s / n; abdev = var = skew = kurt = 0.0; for (i=0; i= 10) { sum = 0; sumsqdev = 0; // sort them into sortedbuf for (i=0; i<10; i++) sortedbuf[i] = buf[i]; qsort(sortedbuf, 10, sizeof(DOUBLE), compare); // use only the inner 8 points in the calculation of mean & var for (i = 1; i < 9; i++) { sum += sortedbuf[i]; } N = 8.0; // using only 8 points mean = sum / N; for (i = 1; i < 9; i++) { sumsqdev += ((sortedbuf[i] - mean) * (sortedbuf[i] - mean)); } sdev = sqrt(sumsqdev / N); if (fabs(y - mean) < 2.5 * sdev) { return FALSE; } else { anomalies++; return TRUE; } } return TRUE; } // AnomalousPoint() VOID AdjustForClockSkew( DOUBLE Slope, DOUBLE Offset) { // // When measuring very low jitter, clock drift between machines // introduces noise in the form of a monotonically increasing // skew between sending and receiving clock. This effect can be // filtered out by finding the best-fit slope for all samples // taken during the calibration period, then using this slope to // normalize the entire run. This routine normalizes using the // slope determined in the routine ClockSkew. // INT i; LOG_RECORD currentRecord; INT64 minLatency = MAX_INT64; INT64 x; DOUBLE mXPlusB; for(i=0; i < g_log.nBuffersLogged; i++){ GetLogEntry(&g_log, ¤tRecord, i); mXPlusB = (currentRecord.TimeSent*Slope) + Offset; // offset is not necessary currentRecord.TimeReceived -= (INT64)mXPlusB; currentRecord.Latency -= (INT64)mXPlusB; SetLogEntry(&g_log, ¤tRecord, i); // // find the minimum latency value // minLatency = (currentRecord.Latency < minLatency)? currentRecord.Latency: minLatency; } for(i=0; i < g_log.nBuffersLogged; i++){ GetLogEntry(&g_log, ¤tRecord, i); currentRecord.Latency -= minLatency; currentRecord.TimeReceived -= minLatency; SetLogEntry(&g_log, ¤tRecord, i); } } // AdjustForClockSkew() #define WACKY 2.5 BOOL FixWackyTimestamps() { // This routine will look over the sender & receiver timestamps and try to see if there // are any non-clock skew related irregularities (such as one of them bumping it's clock // a fixed amount every once-in-a-while) and try to remove them. INT64 *sendstamps, *recvstamps; double *sendgaps, *recvgaps; double *sortedsendgaps, *sortedrecvgaps; double sendmean, sendsdev, sendsum, sendsumsqdev; double recvmean, recvsdev, recvsum, recvsumsqdev; double mediansendgap, medianrecvgap; double modesendgap, moderecvgap; double meansendwackiness, sdevsendwackiness, sumsendwackiness, sumsqdevsendwackiness; double meanrecvwackiness, sdevrecvwackiness, sumrecvwackiness, sumsqdevrecvwackiness; double fractionaldevofsendwackiness, fractionaldevofrecvwackiness; double normalsendgapmean, normalrecvgapmean; double trimmeansendgap, trimmeanrecvgap; BOOL *fWackoSend, *fWackoRecv; int cWackoSend, cWackoRecv; BOOL *fMaybeWackoSend, *fMaybeWackoRecv; int i,N; LOG_RECORD currentRecord; const double FixThreshold = 0.1; double CumulativeFixMagnitude = 0.0; N = (int)g_log.nBuffersLogged; cWackoSend = cWackoRecv = 0; // fill our arrays. sendstamps = malloc(sizeof(INT64) * N); recvstamps = malloc(sizeof(INT64) * N); sendgaps = malloc(sizeof(double) * N); recvgaps = malloc(sizeof(double) * N); sortedsendgaps = malloc(sizeof(double) *N); sortedrecvgaps = malloc(sizeof(double) *N); fWackoRecv = malloc(sizeof(BOOL) * N); fWackoSend = malloc(sizeof(BOOL) * N); fMaybeWackoSend = malloc(sizeof(BOOL) * N); fMaybeWackoRecv = malloc(sizeof(BOOL) * N); for (i=0; i sendmean + WACKY*sendsdev)) { fMaybeWackoSend[i] = fWackoSend[i] = TRUE; } if ((recvgaps[i] < recvmean - WACKY*recvsdev) || (recvgaps[i] > recvmean + WACKY*recvsdev)) { fMaybeWackoRecv[i] = fWackoRecv[i] = TRUE; } } // 3. Check to see if any wacky points are unpaired (that is, a wacky point in the // sending timestamps is not matched with an equally wacky point in the receiving // timestamps). for (i=1; i 0 ) { sinhimControl.sin_addr.s_addr = inet_addr(szHost); } else{ if ((phostent=gethostbyname(szHost)) == NULL) { ErrorExit("bad host name",WSAGetLastError()); } sinhimControl.sin_family = phostent->h_addrtype; memcpy(&(sinhimControl.sin_addr.s_addr), phostent->h_addr, phostent->h_length); } sinhimControl.sin_port = htons(CONTROL_PORT); dwError = bind(sockControl,(SOCKADDR*)&sinmeControl,sizeof(sinmeControl)); } else { // receiver sinmeControl.sin_port = htons(CONTROL_PORT); dwError = bind(sockListen,(SOCKADDR*)&sinmeControl,sizeof(sinmeControl)); } if (dwError == SOCKET_ERROR) ErrorExit("bind failed",WSAGetLastError()); // now connect the socket sinhimControl.sin_family = AF_INET; if (fSender) { // if we're the sender, keep trying to connect until we get through dwAddrSize = MAX_STRING; dwError = WSAAddressToString((SOCKADDR *)&(sinhimControl), sizeof(SOCKADDR_IN), NULL, szAddr, &dwAddrSize); if (dwError == SOCKET_ERROR) ErrorExit("WSAAddressToString failed", WSAGetLastError()); else strcpy(szHisAddr,szAddr); while (TRUE) { dwError = connect(sockControl,(SOCKADDR*)&sinhimControl,sizeof(sinhimControl)); if (!dwError) { printf("control socket: connected to %s\n",szAddr); break; } dwError = WSAGetLastError(); if (dwError != WSAECONNREFUSED) { ErrorExit("connect() failed",dwError); } Sleep(500); // wait a half second between attempts } } else { // if we're the receiver, listen / accept if (listen(sockListen, SOMAXCONN) == SOCKET_ERROR) { ErrorExit("listen() failed", WSAGetLastError()); } sockControl = accept(sockListen, (SOCKADDR*)&sinhimControl, &dwAddrSize); // once we've accepted, close the listen socket closesocket(sockListen); if ((INT_PTR)sockControl < 0) { ErrorExit("accept() failed",WSAGetLastError()); } dwAddrSize = MAX_STRING; dwError = WSAAddressToString((SOCKADDR *)&(sinhimControl), sizeof(SOCKADDR_IN), NULL, szAddr, &dwAddrSize); if (dwError == SOCKET_ERROR) ErrorExit("WSAAddressToString failed", WSAGetLastError()); else strcpy(szHisAddr, szAddr); printf("control socket: accepted connection from %s\n",szAddr); } // set our global control socket variable g_sockControl = sockControl; // record my name dwAddrSize = sizeof(SOCKADDR_IN); getsockname(sockControl,(SOCKADDR *)&(sinmeControl),&dwAddrSize); dwAddrSize = MAX_STRING; dwError = WSAAddressToString((SOCKADDR *)&(sinmeControl), sizeof(SOCKADDR_IN), NULL, szAddr, &dwAddrSize); if (dwError == SOCKET_ERROR) ErrorExit("WSAAddressToString failed", WSAGetLastError()); else strcpy(szMyAddr, szAddr); // exchange version information sprintf(szBuf, "%s %s", MSGST_VER, VERSION_STRING); SendControlMessage(sockControl, szBuf); // now that we're all set, do the actual work of the control socket while (!fDone) { ZeroMemory(szBuf,MAX_STRING); dwError = cbBuf = recv(sockControl, szBuf, MAX_STRING, 0); pchStart = szBuf; pchEnd = szBuf + cbBuf; if (dwError == 0) { // the connection's been gracefully closed fDone = TRUE; closesocket(sockControl); g_fOtherSideFinished=TRUE; ExitThread(0); } if (dwError == SOCKET_ERROR) { dwError = WSAGetLastError(); if (dwError == WSAECONNRESET) { printf("\ncontrol socket: connection reset by peer"); printf("\n\t%I64us since last HELLO packet received", (GetUserTime() - ui64LastHello)/10000000); printf("\n\t%I64us since start", (GetUserTime() - timeStart)/10000000); g_state.Done = TRUE; fDone = TRUE; g_fOtherSideFinished = TRUE; closesocket(sockControl); ExitThread(1); } else { printf("\ncontrol socket: error in recv: %d\n",dwError); g_state.Done = TRUE; fDone = TRUE; g_fOtherSideFinished = TRUE; closesocket(sockControl); ExitThread(1); } continue; } while (pchStart < pchEnd) { ZeroMemory(szCommand,MAX_STRING); // consume the first command and act on it if (pchEnd > szBuf + cbBuf) break; pchEnd = strchr(pchStart, MSGCH_DELIMITER); if (pchEnd == NULL) break; strncpy(szCommand,pchStart,pchEnd - pchStart); if (strcmp(szCommand,MSGST_HELLO) == 0) { // update last hello time ui64LastHello = GetUserTime(); // i should do something like set a timer here that sleeps until a certain timeout // passes, at which point it aborts our transfer } if (strcmp(szCommand,MSGST_ERROR) == 0) { // the other guy's had an error, so we stop and tell him to abort g_fOtherSideFinished = TRUE; g_state.Done = TRUE; fDone = TRUE; SendControlMessage(sockControl,MSGST_ABORT); closesocket(sockControl); ExitThread(1); } if (strcmp(szCommand,MSGST_ABORT) == 0) { // we're told to abort, so do so g_fOtherSideFinished = TRUE; g_state.Done = TRUE; fDone = TRUE; closesocket(sockControl); ExitThread(1); } if (strcmp(szCommand,MSGST_DONE) == 0) { // we're told the other guy's done, so therefore are we closesocket(sockControl); g_fOtherSideFinished = TRUE; g_state.Done = TRUE; fDone = TRUE; ExitThread(1); } if (strcmp(szCommand,MSGST_RSVPERR) == 0) { // we're told the other guy got an rsvp error, so we abort the whole program closesocket(sockControl); g_fOtherSideFinished = TRUE; g_state.Done = TRUE; fDone = TRUE; exit(1); } if (strncmp(szCommand,MSGST_SIZE,4) == 0) { // the sender is telling us how big the buffers are sscanf(szCommand,"%s %d",szT, &g_params.buflen); fGotSize = TRUE; } if (strncmp(szCommand,MSGST_RATE,4) == 0) { // the sender is telling us how fast the buffers are coming sscanf(szCommand, "%s %d",szT, &g_params.TokenRate); fGotRate = TRUE; } if (strncmp(szCommand,MSGST_NUM,3) == 0) { // the sender is telling us how many buffers it's sending sscanf(szCommand, "%s %d",szT, &g_params.nbuf); totalBuffers = g_params.nbuf; fGotNum = TRUE; } if (strncmp(szCommand,MSGST_VER,3) == 0) { sscanf(szCommand, "%s %s",szT, szT2); if (strcmp(szT2,VERSION_STRING) != 0) { printf("WARNING: remote machine using different version of qtcp: %s vs. %s\n", szT2,VERSION_STRING); } } if (trans) { if (strcmp(szCommand,MSGST_READY) == 0) { g_fReadyForXmit = TRUE; } } else { if (!fSentReady && fGotRate && fGotSize && fGotNum) { SendControlMessage(sockControl, MSGST_READY); fSentReady = TRUE; g_fReadyForXmit = TRUE; } } pchStart = pchEnd + 1; pchEnd = szBuf + cbBuf; } } return 0; } int SendControlMessage(SOCKET sock, char * szMsg) { int iResult; char szBuf[MAX_STRING]; sprintf(szBuf,"%s%c",szMsg,MSGCH_DELIMITER); iResult = send (sock, szBuf, strlen(szBuf), 0); if (iResult == SOCKET_ERROR) { return WSAGetLastError(); } return iResult; } void ErrorExit(char *msg, DWORD dwErrorNumber) { fprintf(stderr,"ERROR: %d\n",dwErrorNumber); if (msg != NULL) fprintf(stderr,"\t%s\n",msg); else { switch(dwErrorNumber) { case WSAEFAULT: fprintf(stderr,"\tWSAEFAULT: Buffer too small to contain name\n"); break; case WSAEINVAL: fprintf(stderr,"\tWSAEINVAL: Invalid socket address\n"); break; case WSANOTINITIALISED: fprintf(stderr,"\tWSANOTINITIALIZED: WSA Not initialized\n"); break; default: fprintf(stderr,"\tUnknown error\n"); break; } } SendControlMessage(g_sockControl, MSGST_ABORT); DestroyLog(&g_log); WSACleanup(); exit(1); _exit(1); } // some math utility functions // comparison for doubles (to use in qsort) int __cdecl compare( const void *arg1, const void *arg2 ) { DOUBLE dTemp; DOUBLE d1 = * (DOUBLE *) arg1; DOUBLE d2 = * (DOUBLE *) arg2; dTemp = d1 - d2; if (dTemp < 0) return -1; if (dTemp == 0) return 0; else return 1; } // comparison for ints (to use in qsort) int __cdecl compareint( const void *arg1, const void *arg2 ) { int nTemp; int n1 = * (int *) arg1; int n2 = * (int *) arg2; nTemp = n1 - n2; if (nTemp < 0) return -1; if (nTemp == 0) return 0; else return 1; } // comparison for int64s (to use in qsort) int __cdecl compareI64( const void *arg1, const void *arg2 ) { INT64 nTemp; INT64 n1 = * (INT64 *) arg1; INT64 n2 = * (INT64 *) arg2; nTemp = n1 - n2; if (nTemp < 0) return -1; if (nTemp == 0) return 0; else return 1; } #define EPS 1.0e-7 // sum up error function for given value of b double rofunc(double b, int N, double yt[], double xt[], double * paa, double * pabdevt) { int i; double *pfT; double d, sum=0.0; double aa = *paa; double abdevt = *pabdevt; pfT = malloc(sizeof(double) * N); for (i = 0; i < N; i++) pfT[i] = yt[i]-b*xt[i]; qsort(pfT, N, sizeof(DOUBLE), compare); if (N & 1) { // odd N aa = pfT[(N+1) / 2]; } else { i = N / 2; aa = 0.5 * (pfT[i] + pfT[i+1]); } abdevt = 0.0; for (i = 0; i EPS) sum += (d >= 0.0 ? xt[i]: -xt[i]); } *paa = aa; *pabdevt = abdevt; free(pfT); return sum; } #define SIGN(a,b) ((b) >= 0 ? fabs(a) : fabs(-a)) void medfit(double x[], double y[], int N, double *a, double *b, double *abdev) { // fit y = a + bx to least absolute deviation. abdev is mean absolute deviation. // incoming, a and b are treated as starting guesses int i; double *xt = x; double *yt = y; double sx, sy, sxy, sxx, chisq; double del, sigb; double bb, b1, b2, aa, abdevt, f, f1, f2, temp; sx = sy = sxy = sxx = chisq = 0.0; // we find chisq fit to use as starting guess for (i=0; i 0.0) { if (fabs(f1) < fabs(f2)) f1 = rofunc(b1 += 1.6*(b1-b2),N,yt,xt,&aa,&abdevt); else f2 = rofunc(b2 += 1.6*(b2-b1),N,yt,xt,&aa,&abdevt); } sigb = 0.000001 * sigb; // refine while (fabs(b2 - b1) > sigb) { bb = b1 + 0.5 * (b2 - b1); if (bb == b1 || bb == b2) break; f = rofunc(bb, N, yt, xt, &aa, &abdevt); if (f*f1 >= 0.0) { f1 = f; b1 = bb; } else { f2 = f; b2 = bb; } } } *a = aa; *b = bb; *abdev = abdevt / N; } double mode(const double data[], const int N) { // finds and returns the mode of the N points in data double * sorted; double mode, cur=0; int cMode, cCur; int i; sorted = malloc(N * sizeof(double)); for (i=0; i cMode) { mode = cur; cMode = cCur; } if (sorted[i] == mode) { cMode++; } else { if (sorted[i] == cur) cCur++; else { cur = sorted[i]; cCur = 1; } } } free(sorted); return mode; } void RemoveDuplicates(int rg[], int * pN) { // this removes duplicates from the array passed in and returns it with *pN = #remaining // it makes no guarantees about elements after rg[#remaining] int *pNewArray; int cNew; int i; qsort(rg,*pN,sizeof(int),compareint); pNewArray = malloc(sizeof(int) * *pN); pNewArray[0] = rg[0]; cNew = 1; for (i=1; i<*pN; i++) { if (rg[i] != pNewArray[cNew - 1]) { pNewArray[cNew++] = rg[i]; } } *pN = cNew; for (i=0; iTokenRate); printf("TokenBucketSize: %lu bytes\n",lpfs->TokenBucketSize); printf("PeakBandwidth: %lu bytes/sec\n",lpfs->PeakBandwidth); printf("Latency: %lu microseconds\n",lpfs->Latency); printf("DelayVariation: %lu microseconds\n",lpfs->DelayVariation); printf("ServiceType: %X\n",lpfs->ServiceType); printf("MaxSduSize: %lu bytes\n",lpfs->MaxSduSize); printf("MinimumPolicedSize: %lu bytes\n",lpfs->MinimumPolicedSize); }