datapath-windows/ovsext/OvsUser.c

   1 /*
   2  * Copyright (c) 2014 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * OvsUser.c
  19  *      Manage packet queue for packet miss for userAction.
  20  */
  21
  22
  23 #include "precomp.h"
  24
  25 #include "Datapath.h"
  26 #include "OvsSwitch.h"
  27 #include "OvsVport.h"
  28 #include "OvsEvent.h"
  29 #include "OvsUser.h"
  30 #include "OvsPacketIO.h"
  31 #include "OvsChecksum.h"
  32 #include "OvsNetProto.h"
  33 #include "OvsFlow.h"
  34 #include "OvsTunnelIntf.h"
  35
  36 #ifdef OVS_DBG_MOD
  37 #undef OVS_DBG_MOD
  38 #endif
  39 #define OVS_DBG_MOD OVS_DBG_USER
  40 #include "OvsDebug.h"
  41
  42 OVS_USER_PACKET_QUEUE ovsPacketQueues[OVS_MAX_NUM_PACKET_QUEUES];
  43
  44 POVS_PACKET_QUEUE_ELEM OvsGetNextPacket(POVS_OPEN_INSTANCE instance);
  45 extern PNDIS_SPIN_LOCK gOvsCtrlLock;
  46 extern POVS_SWITCH_CONTEXT gOvsSwitchContext;
  47 OVS_USER_STATS ovsUserStats;
  48
  49
  50 NTSTATUS
  51 OvsUserInit()
  52 {
  53     UINT32 i;
  54     POVS_USER_PACKET_QUEUE queue;
  55     for (i = 0; i < OVS_MAX_NUM_PACKET_QUEUES; i++) {
  56         queue = &ovsPacketQueues[i];
  57         RtlZeroMemory(queue, sizeof (*queue));
  58         InitializeListHead(&queue->packetList);
  59         NdisAllocateSpinLock(&queue->queueLock);
  60     }
  61     return STATUS_SUCCESS;
  62 }
  63
  64 VOID
  65 OvsUserCleanup()
  66 {
  67     UINT32 i;
  68     POVS_USER_PACKET_QUEUE queue;
  69     for (i = 0; i < OVS_MAX_NUM_PACKET_QUEUES; i++) {
  70         queue = &ovsPacketQueues[i];
  71         ASSERT(IsListEmpty(&queue->packetList));
  72         ASSERT(queue->instance == NULL);
  73         ASSERT(queue->pendingIrp == NULL);
  74         NdisFreeSpinLock(&queue->queueLock);
  75     }
  76 }
  77
  78 static VOID
  79 OvsPurgePacketQueue(POVS_USER_PACKET_QUEUE queue,
  80                     POVS_OPEN_INSTANCE instance)
  81 {
  82     PLIST_ENTRY link, next;
  83     LIST_ENTRY tmp;
  84     POVS_PACKET_QUEUE_ELEM elem;
  85
  86     InitializeListHead(&tmp);
  87     NdisAcquireSpinLock(&queue->queueLock);
  88     if (queue->instance != instance) {
  89         NdisReleaseSpinLock(&queue->queueLock);
  90         return;
  91     }
  92
  93     if (queue->numPackets) {
  94         OvsAppendList(&tmp, &queue->packetList);
  95         queue->numPackets = 0;
  96     }
  97     NdisReleaseSpinLock(&queue->queueLock);
  98     LIST_FORALL_SAFE(&tmp, link, next) {
  99         RemoveEntryList(link);
 100         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 101         OvsFreeMemory(elem);
 102     }
 103 }
 104
 105
 106 VOID
 107 OvsCleanupPacketQueue(POVS_OPEN_INSTANCE instance)
 108 {
 109     POVS_USER_PACKET_QUEUE queue;
 110     POVS_PACKET_QUEUE_ELEM elem;
 111     PLIST_ENTRY link, next;
 112     LIST_ENTRY tmp;
 113     PIRP irp = NULL;
 114
 115     InitializeListHead(&tmp);
 116     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 117     if (queue) {
 118         PDRIVER_CANCEL cancelRoutine;
 119         NdisAcquireSpinLock(&queue->queueLock);
 120         if (queue->instance != instance) {
 121             NdisReleaseSpinLock(&queue->queueLock);
 122             return;
 123         }
 124
 125         if (queue->numPackets) {
 126             OvsAppendList(&tmp, &queue->packetList);
 127             queue->numPackets = 0;
 128         }
 129         queue->instance = NULL;
 130         queue->queueId = OVS_MAX_NUM_PACKET_QUEUES;
 131         instance->packetQueue = NULL;
 132         irp = queue->pendingIrp;
 133         queue->pendingIrp = NULL;
 134         if (irp) {
 135             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 136             if (cancelRoutine == NULL) {
 137                 irp = NULL;
 138             }
 139         }
 140         NdisReleaseSpinLock(&queue->queueLock);
 141     }
 142     LIST_FORALL_SAFE(&tmp, link, next) {
 143         RemoveEntryList(link);
 144         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 145         OvsFreeMemory(elem);
 146     }
 147     if (irp) {
 148         OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 149     }
 150 }
 151
 152 NTSTATUS
 153 OvsSubscribeDpIoctl(PFILE_OBJECT fileObject,
 154                     PVOID inputBuffer,
 155                     UINT32 inputLength)
 156 {
 157     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 158     UINT32 queueId;
 159     POVS_USER_PACKET_QUEUE queue;
 160     if (inputLength < sizeof (UINT32)) {
 161         return STATUS_INVALID_PARAMETER;
 162     }
 163     queueId = *(UINT32 *)inputBuffer;
 164     if (instance->packetQueue && queueId >= OVS_MAX_NUM_PACKET_QUEUES) {
 165         /*
 166          * unsubscribe
 167          */
 168         OvsCleanupPacketQueue(instance);
 169     } else if (instance->packetQueue == NULL &&
 170                queueId < OVS_MAX_NUM_PACKET_QUEUES) {
 171         queue = &ovsPacketQueues[queueId];
 172         NdisAcquireSpinLock(&queue->queueLock);
 173         if (ovsPacketQueues[queueId].instance) {
 174              if (ovsPacketQueues[queueId].instance != instance) {
 175                  NdisReleaseSpinLock(&queue->queueLock);
 176                  return STATUS_INSUFFICIENT_RESOURCES;
 177              } else {
 178                  NdisReleaseSpinLock(&queue->queueLock);
 179                  return STATUS_SUCCESS;
 180              }
 181         }
 182         queue->queueId = queueId;
 183         queue->instance = instance;
 184         instance->packetQueue = queue;
 185         ASSERT(IsListEmpty(&queue->packetList));
 186         NdisReleaseSpinLock(&queue->queueLock);
 187     } else {
 188         return STATUS_INVALID_PARAMETER;
 189     }
 190     return STATUS_SUCCESS;
 191 }
 192
 193
 194 NTSTATUS
 195 OvsReadDpIoctl(PFILE_OBJECT fileObject,
 196                PVOID outputBuffer,
 197                UINT32 outputLength,
 198                UINT32 *replyLen)
 199 {
 200     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 201     POVS_PACKET_QUEUE_ELEM elem;
 202     UINT32 len;
 203
 204 #define TCP_CSUM_OFFSET  16
 205 #define UDP_CSUM_OFFSET  6
 206     ASSERT(instance);
 207
 208     if (instance->packetQueue == NULL) {
 209         return STATUS_INVALID_PARAMETER;
 210     }
 211     if (outputLength < (sizeof (OVS_PACKET_INFO) + OVS_MIN_PACKET_SIZE)) {
 212         return STATUS_BUFFER_TOO_SMALL;
 213     }
 214
 215     elem = OvsGetNextPacket(instance);
 216     if (elem) {
 217         /*
 218          * XXX revisit this later
 219          */
 220         len = elem->packet.totalLen > outputLength ? outputLength :
 221                  elem->packet.totalLen;
 222
 223         if ((elem->hdrInfo.tcpCsumNeeded || elem->hdrInfo.udpCsumNeeded) &&
 224             len == elem->packet.totalLen) {
 225             UINT16 sum, *ptr;
 226             UINT16 size = (UINT16)(elem->packet.userDataLen +
 227                                    elem->hdrInfo.l4Offset +
 228                                    (UINT16)sizeof (OVS_PACKET_INFO));
 229             RtlCopyMemory(outputBuffer, &elem->packet, size);
 230             ASSERT(len - size >=  elem->hdrInfo.l4PayLoad);
 231             sum = CopyAndCalculateChecksum((UINT8 *)outputBuffer + size,
 232                                            (UINT8 *)&elem->packet + size,
 233                                            elem->hdrInfo.l4PayLoad, 0);
 234             ptr =(UINT16 *)((UINT8 *)outputBuffer + size +
 235                             (elem->hdrInfo.tcpCsumNeeded ?
 236                              TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 237             *ptr = sum;
 238             ovsUserStats.l4Csum++;
 239         } else {
 240             RtlCopyMemory(outputBuffer, &elem->packet, len);
 241         }
 242
 243         *replyLen = len;
 244         OvsFreeMemory(elem);
 245     }
 246     return STATUS_SUCCESS;
 247 }
 248
 249 /* Helper function to allocate a Forwarding Context for an NBL */
 250 NTSTATUS
 251 OvsAllocateForwardingContextForNBL(POVS_SWITCH_CONTEXT switchContext,
 252                                    PNET_BUFFER_LIST nbl)
 253 {
 254     return switchContext->NdisSwitchHandlers.
 255         AllocateNetBufferListForwardingContext(
 256             switchContext->NdisSwitchContext, nbl);
 257 }
 258
 259 /*
 260  * --------------------------------------------------------------------------
 261  * This function allocates all the stuff necessary for creating an NBL from the
 262  * input buffer of specified length, namely, a nonpaged data buffer of size
 263  * length, an MDL from it, and a NB and NBL from it. It does not allocate an NBL
 264  * context yet. It also copies data from the specified buffer to the NBL.
 265  * --------------------------------------------------------------------------
 266  */
 267 PNET_BUFFER_LIST
 268 OvsAllocateNBLForUserBuffer(POVS_SWITCH_CONTEXT switchContext,
 269                             PVOID userBuffer,
 270                             ULONG length)
 271 {
 272     UINT8 *data = NULL;
 273     PNET_BUFFER_LIST nbl = NULL;
 274     PNET_BUFFER nb;
 275     PMDL mdl;
 276
 277     if (length > OVS_DEFAULT_DATA_SIZE) {
 278         nbl = OvsAllocateVariableSizeNBL(switchContext, length,
 279                                          OVS_DEFAULT_HEADROOM_SIZE);
 280
 281     } else {
 282         nbl = OvsAllocateFixSizeNBL(switchContext, length,
 283                                     OVS_DEFAULT_HEADROOM_SIZE);
 284     }
 285     if (nbl == NULL) {
 286         return NULL;
 287     }
 288
 289     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 290     mdl = NET_BUFFER_CURRENT_MDL(nb);
 291     data = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority) +
 292                     NET_BUFFER_CURRENT_MDL_OFFSET(nb);
 293     if (!data) {
 294         OvsCompleteNBL(switchContext, nbl, TRUE);
 295         return NULL;
 296     }
 297
 298     NdisMoveMemory(data, userBuffer, length);
 299
 300     return nbl;
 301 }
 302
 303 NTSTATUS
 304 OvsExecuteDpIoctl(PVOID inputBuffer,
 305                   UINT32 inputLength,
 306                   UINT32 outputLength)
 307 {
 308     NTSTATUS                    status = STATUS_SUCCESS;
 309     NTSTATUS                    ndisStatus;
 310     OvsPacketExecute            *execute;
 311     LOCK_STATE_EX               lockState;
 312     PNET_BUFFER_LIST pNbl;
 313     struct nlattr *actions;
 314     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 315     OvsFlowKey key;
 316     OVS_PACKET_HDR_INFO layers;
 317     POVS_VPORT_ENTRY vport;
 318
 319     if (inputLength < sizeof(*execute) || outputLength != 0) {
 320         return STATUS_INFO_LENGTH_MISMATCH;
 321     }
 322
 323     NdisAcquireSpinLock(gOvsCtrlLock);
 324     if (gOvsSwitchContext == NULL) {
 325         status = STATUS_INVALID_PARAMETER;
 326         goto unlock;
 327     }
 328
 329     execute = (struct OvsPacketExecute *) inputBuffer;
 330
 331     if (execute->packetLen == 0) {
 332         status = STATUS_INVALID_PARAMETER;
 333         goto unlock;
 334     }
 335
 336     if (inputLength != sizeof (*execute) +
 337                        execute->actionsLen + execute->packetLen) {
 338         status = STATUS_INFO_LENGTH_MISMATCH;
 339         goto unlock;
 340     }
 341     actions = (struct nlattr *)((PCHAR)&execute->actions + execute->packetLen);
 342
 343     /*
 344      * Allocate the NBL, copy the data from the userspace buffer. Allocate
 345      * also, the forwarding context for the packet.
 346      */
 347     pNbl = OvsAllocateNBLForUserBuffer(gOvsSwitchContext, &execute->packetBuf,
 348                                        execute->packetLen);
 349     if (pNbl == NULL) {
 350         status = STATUS_NO_MEMORY;
 351         goto unlock;
 352     }
 353
 354     fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(pNbl);
 355     vport = OvsFindVportByPortNo(gOvsSwitchContext, execute->inPort);
 356     if (vport) {
 357         fwdDetail->SourcePortId = vport->portId;
 358         fwdDetail->SourceNicIndex = vport->nicIndex;
 359     } else {
 360         fwdDetail->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 361         fwdDetail->SourceNicIndex = 0;
 362     }
 363     // XXX: Figure out if any of the other members of fwdDetail need to be set.
 364
 365     ndisStatus = OvsExtractFlow(pNbl, fwdDetail->SourcePortId, &key, &layers,
 366                               NULL);
 367     if (ndisStatus == NDIS_STATUS_SUCCESS) {
 368         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 369         NdisAcquireRWLockRead(gOvsSwitchContext->dispatchLock, &lockState,
 370                               NDIS_RWL_AT_DISPATCH_LEVEL);
 371         ndisStatus = OvsActionsExecute(gOvsSwitchContext, NULL, pNbl,
 372                                        vport ? vport->portNo : 0,
 373                                        NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP,
 374                                        &key, NULL, &layers, actions,
 375                                        execute->actionsLen);
 376         pNbl = NULL;
 377         NdisReleaseRWLock(gOvsSwitchContext->dispatchLock, &lockState);
 378     }
 379     if (ndisStatus != NDIS_STATUS_SUCCESS) {
 380         status = STATUS_UNSUCCESSFUL;
 381     }
 382
 383     if (pNbl) {
 384         OvsCompleteNBL(gOvsSwitchContext, pNbl, TRUE);
 385     }
 386 unlock:
 387     NdisReleaseSpinLock(gOvsCtrlLock);
 388     return status;
 389 }
 390
 391
 392 NTSTATUS
 393 OvsPurgeDpIoctl(PFILE_OBJECT fileObject)
 394 {
 395     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 396     POVS_USER_PACKET_QUEUE queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 397
 398     if (queue == NULL) {
 399         return STATUS_INVALID_PARAMETER;
 400     }
 401     OvsPurgePacketQueue(queue, instance);
 402     return STATUS_SUCCESS;
 403 }
 404
 405 VOID
 406 OvsCancelIrpDatapath(PDEVICE_OBJECT deviceObject,
 407                      PIRP irp)
 408 {
 409     PIO_STACK_LOCATION irpSp;
 410     PFILE_OBJECT fileObject;
 411     POVS_OPEN_INSTANCE instance;
 412     POVS_USER_PACKET_QUEUE queue = NULL;
 413
 414     UNREFERENCED_PARAMETER(deviceObject);
 415
 416     IoReleaseCancelSpinLock(irp->CancelIrql);
 417     irpSp = IoGetCurrentIrpStackLocation(irp);
 418     fileObject = irpSp->FileObject;
 419
 420     if (fileObject == NULL) {
 421         goto done;
 422     }
 423     NdisAcquireSpinLock(gOvsCtrlLock);
 424     instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 425     if (instance) {
 426         queue = instance->packetQueue;
 427     }
 428     if (instance == NULL || queue == NULL) {
 429         NdisReleaseSpinLock(gOvsCtrlLock);
 430         goto done;
 431     }
 432     NdisReleaseSpinLock(gOvsCtrlLock);
 433     NdisAcquireSpinLock(&queue->queueLock);
 434     if (queue->pendingIrp == irp) {
 435         queue->pendingIrp = NULL;
 436     }
 437     NdisReleaseSpinLock(&queue->queueLock);
 438 done:
 439     OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 440 }
 441
 442
 443 NTSTATUS
 444 OvsWaitDpIoctl(PIRP irp, PFILE_OBJECT fileObject)
 445 {
 446     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 447     POVS_USER_PACKET_QUEUE queue =
 448                (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 449     NTSTATUS status = STATUS_SUCCESS;
 450     BOOLEAN cancelled = FALSE;
 451
 452     if (queue == NULL) {
 453         return STATUS_INVALID_PARAMETER;
 454     }
 455     NdisAcquireSpinLock(&queue->queueLock);
 456     if (queue->instance != instance) {
 457         NdisReleaseSpinLock(&queue->queueLock);
 458         return STATUS_INVALID_PARAMETER;
 459     }
 460     if (queue->pendingIrp) {
 461         NdisReleaseSpinLock(&queue->queueLock);
 462         return STATUS_DEVICE_BUSY;
 463     }
 464     if (queue->numPackets == 0) {
 465         PDRIVER_CANCEL cancelRoutine;
 466         IoMarkIrpPending(irp);
 467         IoSetCancelRoutine(irp, OvsCancelIrpDatapath);
 468         if (irp->Cancel) {
 469             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 470             if (cancelRoutine) {
 471                 cancelled = TRUE;
 472             }
 473         } else {
 474             queue->pendingIrp = irp;
 475         }
 476         status = STATUS_PENDING;
 477     }
 478     NdisReleaseSpinLock(&queue->queueLock);
 479     if (cancelled) {
 480         OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 481         OVS_LOG_INFO("Datapath IRP cancelled: %p", irp);
 482     }
 483     return status;
 484 }
 485
 486
 487 POVS_PACKET_QUEUE_ELEM
 488 OvsGetNextPacket(POVS_OPEN_INSTANCE instance)
 489 {
 490     POVS_USER_PACKET_QUEUE queue;
 491     PLIST_ENTRY link;
 492     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 493     if (queue == NULL) {
 494         return NULL;
 495     }
 496     NdisAcquireSpinLock(&queue->queueLock);
 497     if (queue->instance != instance || queue->numPackets == 0) {
 498         NdisReleaseSpinLock(&queue->queueLock);
 499         return NULL;
 500     }
 501     link = RemoveHeadList(&queue->packetList);
 502     queue->numPackets--;
 503     NdisReleaseSpinLock(&queue->queueLock);
 504     return CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 505 }
 506
 507
 508 POVS_USER_PACKET_QUEUE
 509 OvsGetQueue(UINT32 queueId)
 510 {
 511     POVS_USER_PACKET_QUEUE queue;
 512     if (queueId >= OVS_MAX_NUM_PACKET_QUEUES) {
 513         return NULL;
 514     }
 515     queue = &ovsPacketQueues[queueId];
 516     return queue->instance != NULL ? queue : NULL;
 517 }
 518
 519 /*
 520  *----------------------------------------------------------------------------
 521  * OvsCreateQueuePacket --
 522  *
 523  *  Create a packet which will be forwarded to user space.
 524  *
 525  * InputParameter:
 526  *   queueId Identify the queue the packet to be inserted
 527  *      This will be used when multiple queues is supported
 528  *      in userspace
 529  *   userData: when cmd is user action, this field contain
 530  *      user action data.
 531  *   userDataLen: as name indicated
 532  *   cmd: either miss or user action
 533  *   inPort: datapath port id from which the packet is received.
 534  *   tunnelKey: tunnelKey for tunneled packet
 535  *   nbl:  the NET_BUFFER_LIST which contain the packet
 536  *   nb: the packet
 537  *   isRecv: This is used to decide how to interprete the csum info
 538  *   hdrInfo: include hdr info initialized during flow extraction.
 539  *
 540  * Results:
 541  *    NULL if fail to create the packet
 542  *    The packet element otherwise
 543  *----------------------------------------------------------------------------
 544  */
 545 POVS_PACKET_QUEUE_ELEM
 546 OvsCreateQueuePacket(UINT32 queueId,
 547                      PVOID userData,
 548                      UINT32 userDataLen,
 549                      UINT32 cmd,
 550                      UINT32 inPort,
 551                      OvsIPv4TunnelKey *tunnelKey,
 552                      PNET_BUFFER_LIST nbl,
 553                      PNET_BUFFER nb,
 554                      BOOLEAN isRecv,
 555                      POVS_PACKET_HDR_INFO hdrInfo)
 556 {
 557 #define VLAN_TAG_SIZE 4
 558     UINT32 allocLen, dataLen, extraLen = 0;
 559     POVS_PACKET_QUEUE_ELEM elem;
 560     PMDL mdl;
 561     UINT8 *src, *dst;
 562     NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csumInfo;
 563     NDIS_NET_BUFFER_LIST_8021Q_INFO vlanInfo;
 564
 565     csumInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo);
 566
 567     if (isRecv && (csumInfo.Receive.TcpChecksumFailed ||
 568                    (csumInfo.Receive.UdpChecksumFailed &&
 569                     !hdrInfo->udpCsumZero) ||
 570                    csumInfo.Receive.IpChecksumFailed)) {
 571         OVS_LOG_INFO("Packet dropped due to checksum failure.");
 572         ovsUserStats.dropDuetoChecksum++;
 573         return NULL;
 574     }
 575
 576     vlanInfo.Value = NET_BUFFER_LIST_INFO(nbl, Ieee8021QNetBufferListInfo);
 577     if (vlanInfo.TagHeader.VlanId) {
 578         /*
 579          * We may also need to check priority XXX
 580          */
 581         extraLen = VLAN_TAG_SIZE;
 582     }
 583
 584     dataLen = NET_BUFFER_DATA_LENGTH(nb);
 585     allocLen = sizeof (OVS_PACKET_QUEUE_ELEM) + userDataLen + dataLen +
 586            extraLen;
 587
 588     elem = (POVS_PACKET_QUEUE_ELEM)OvsAllocateMemory(allocLen);
 589     if (elem == NULL) {
 590         ovsUserStats.dropDuetoResource++;
 591         return NULL;
 592     }
 593     elem->hdrInfo.value = hdrInfo->value;
 594     elem->packet.totalLen = sizeof (OVS_PACKET_INFO) + userDataLen + dataLen +
 595        extraLen;
 596     elem->packet.queue = queueId;
 597     elem->packet.userDataLen = userDataLen;
 598     elem->packet.inPort = inPort;
 599     elem->packet.cmd = cmd;
 600     if (cmd == (UINT32)OVS_PACKET_CMD_MISS) {
 601         ovsUserStats.miss++;
 602     } else {
 603         ovsUserStats.action++;
 604     }
 605     elem->packet.packetLen = dataLen + extraLen;
 606     if (tunnelKey) {
 607         RtlCopyMemory(&elem->packet.tunnelKey, tunnelKey,
 608                       sizeof (*tunnelKey));
 609     } else {
 610         RtlZeroMemory(&elem->packet.tunnelKey,
 611                       sizeof (elem->packet.tunnelKey));
 612     }
 613
 614     dst = elem->packet.data;
 615     if (userDataLen) {
 616         RtlCopyMemory(dst, userData, userDataLen);
 617         dst = dst + userDataLen;
 618     }
 619     dst += extraLen;
 620
 621     mdl = NET_BUFFER_CURRENT_MDL(nb);
 622     src = NdisGetDataBuffer(nb, dataLen, dst, 1, 0);
 623     if (src == NULL) {
 624         OvsFreeMemory(elem);
 625         ovsUserStats.dropDuetoResource++;
 626         return NULL;
 627     } else if (src != dst) {
 628         /* Copy the data from the NDIS buffer to dst. */
 629         RtlCopyMemory(dst, src, dataLen);
 630     }
 631
 632     dst =  elem->packet.data + userDataLen + extraLen;
 633     /*
 634      * Fix IP hdr if necessary
 635      */
 636     if ((isRecv && csumInfo.Receive.IpChecksumValueInvalid) ||
 637         (!isRecv && csumInfo.Transmit.IsIPv4 &&
 638          csumInfo.Transmit.IpHeaderChecksum)) {
 639         PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 640         ASSERT(elem->hdrInfo.isIPv4);
 641         ASSERT(ipHdr->Version == 4);
 642         ipHdr->HeaderChecksum = IPChecksum((UINT8 *)ipHdr,
 643                                            ipHdr->HeaderLength << 2,
 644                                            (UINT16)~ipHdr->HeaderChecksum);
 645         ovsUserStats.ipCsum++;
 646     }
 647     ASSERT(elem->hdrInfo.tcpCsumNeeded == 0 &&
 648            elem->hdrInfo.udpCsumNeeded == 0);
 649     /*
 650      * Fow now, we will not do verification
 651      * There is no correctness issue here.
 652      * XXX
 653      */
 654     /*
 655      * calculate TCP/UDP pseudo checksum
 656      */
 657     if (isRecv && csumInfo.Receive.TcpChecksumValueInvalid) {
 658         /*
 659          * Only this case, we need to reclaculate pseudo checksum
 660          * all other cases, it is assumed the pseudo checksum is
 661          * filled already.
 662          *
 663          */
 664         PTCP_HDR tcpHdr = (PTCP_HDR)(dst + hdrInfo->l4Offset);
 665         if (hdrInfo->isIPv4) {
 666             PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 667             elem->hdrInfo.l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 668                                                (ipHdr->HeaderLength << 2));
 669             tcpHdr->th_sum =
 670                  IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 671                                   (UINT32 *)&ipHdr->DestinationAddress,
 672                                   IPPROTO_TCP, elem->hdrInfo.l4PayLoad);
 673         } else {
 674             PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(dst + hdrInfo->l3Offset);
 675             elem->hdrInfo.l4PayLoad =
 676                   (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 677                            hdrInfo->l3Offset + sizeof(IPV6_HEADER) -
 678                            hdrInfo->l4Offset);
 679             ASSERT(hdrInfo->isIPv6);
 680             tcpHdr->th_sum =
 681                 IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 682                                    (UINT32 *)&ipv6Hdr->DestinationAddress,
 683                                    IPPROTO_TCP, elem->hdrInfo.l4PayLoad);
 684         }
 685         elem->hdrInfo.tcpCsumNeeded = 1;
 686         ovsUserStats.recalTcpCsum++;
 687     } else if (!isRecv) {
 688         if (csumInfo.Transmit.TcpChecksum) {
 689             elem->hdrInfo.tcpCsumNeeded = 1;
 690         } else if (csumInfo.Transmit.UdpChecksum) {
 691             elem->hdrInfo.udpCsumNeeded = 1;
 692         }
 693         if (elem->hdrInfo.tcpCsumNeeded || elem->hdrInfo.udpCsumNeeded) {
 694 #ifdef DBG
 695             UINT16 sum, *ptr;
 696             UINT8 proto =
 697                elem->hdrInfo.tcpCsumNeeded ? IPPROTO_TCP : IPPROTO_UDP;
 698 #endif
 699             if (hdrInfo->isIPv4) {
 700                 PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 701                 elem->hdrInfo.l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 702                                                    (ipHdr->HeaderLength << 2));
 703 #ifdef DBG
 704                 sum = IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 705                                        (UINT32 *)&ipHdr->DestinationAddress,
 706                                        proto, elem->hdrInfo.l4PayLoad);
 707 #endif
 708             } else {
 709                 PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(dst +
 710                                                       hdrInfo->l3Offset);
 711                 elem->hdrInfo.l4PayLoad =
 712                        (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 713                                 hdrInfo->l3Offset + sizeof(IPV6_HEADER) -
 714                                 hdrInfo->l4Offset);
 715                 ASSERT(hdrInfo->isIPv6);
 716 #ifdef DBG
 717                 sum = IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 718                                          (UINT32 *)&ipv6Hdr->DestinationAddress,
 719                                          proto, elem->hdrInfo.l4PayLoad);
 720 #endif
 721             }
 722 #ifdef DBG
 723             ptr = (UINT16 *)(dst + hdrInfo->l4Offset +
 724                              (elem->hdrInfo.tcpCsumNeeded ?
 725                               TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 726             ASSERT(*ptr == sum);
 727 #endif
 728         }
 729     }
 730     /*
 731      * Finally insert VLAN tag
 732      */
 733     if (extraLen) {
 734         dst = elem->packet.data + userDataLen;
 735         src = dst + extraLen;
 736         ((UINT32 *)dst)[0] = ((UINT32 *)src)[0];
 737         ((UINT32 *)dst)[1] = ((UINT32 *)src)[1];
 738         ((UINT32 *)dst)[2] = ((UINT32 *)src)[2];
 739         dst += 12;
 740         ((UINT16 *)dst)[0] = htons(0x8100);
 741         ((UINT16 *)dst)[1] = htons(vlanInfo.TagHeader.VlanId |
 742                                    (vlanInfo.TagHeader.UserPriority << 13));
 743         elem->hdrInfo.l3Offset += VLAN_TAG_SIZE;
 744         elem->hdrInfo.l4Offset += VLAN_TAG_SIZE;
 745         ovsUserStats.vlanInsert++;
 746     }
 747
 748     return elem;
 749 }
 750
 751
 752 VOID
 753 OvsQueuePackets(UINT32 queueId,
 754                 PLIST_ENTRY packetList,
 755                 UINT32 numElems)
 756 {
 757     POVS_USER_PACKET_QUEUE queue = OvsGetQueue(queueId);
 758     POVS_PACKET_QUEUE_ELEM elem;
 759     PIRP irp = NULL;
 760     PLIST_ENTRY  link;
 761     UINT32 num = 0;
 762
 763     OVS_LOG_LOUD("Enter: queueId %u, numELems: %u",
 764                   queueId, numElems);
 765     if (queue == NULL) {
 766         goto cleanup;
 767     }
 768
 769     NdisAcquireSpinLock(&queue->queueLock);
 770     if (queue->instance == NULL) {
 771         NdisReleaseSpinLock(&queue->queueLock);
 772         goto cleanup;
 773     } else {
 774         OvsAppendList(&queue->packetList, packetList);
 775         queue->numPackets += numElems;
 776     }
 777     if (queue->pendingIrp) {
 778         PDRIVER_CANCEL cancelRoutine;
 779         irp = queue->pendingIrp;
 780         queue->pendingIrp = NULL;
 781         cancelRoutine = IoSetCancelRoutine(irp, NULL);
 782         if (cancelRoutine == NULL) {
 783             irp = NULL;
 784         }
 785     }
 786     NdisReleaseSpinLock(&queue->queueLock);
 787     if (irp) {
 788         OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 789     }
 790
 791 cleanup:
 792     while (!IsListEmpty(packetList)) {
 793         link = RemoveHeadList(packetList);
 794         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 795         OvsFreeMemory(elem);
 796         num++;
 797     }
 798     OVS_LOG_LOUD("Exit: drop %u packets", num);
 799 }
 800
 801
 802 /*
 803  *----------------------------------------------------------------------------
 804  * OvsCreateAndAddPackets --
 805  *
 806  *  Create a packet and forwarded to user space.
 807  *
 808  *  This function would fragment packet if needed, and queue
 809  *  each segment to user space.
 810  *----------------------------------------------------------------------------
 811  */
 812 NTSTATUS
 813 OvsCreateAndAddPackets(UINT32 queueId,
 814                        PVOID userData,
 815                        UINT32 userDataLen,
 816                        UINT32 cmd,
 817                        UINT32 inPort,
 818                        OvsIPv4TunnelKey *tunnelKey,
 819                        PNET_BUFFER_LIST nbl,
 820                        BOOLEAN isRecv,
 821                        POVS_PACKET_HDR_INFO hdrInfo,
 822                        POVS_SWITCH_CONTEXT switchContext,
 823                        LIST_ENTRY *list,
 824                        UINT32 *num)
 825 {
 826     POVS_PACKET_QUEUE_ELEM elem;
 827     PNET_BUFFER_LIST newNbl = NULL;
 828     PNET_BUFFER nb;
 829
 830     if (hdrInfo->isTcp) {
 831         NDIS_TCP_LARGE_SEND_OFFLOAD_NET_BUFFER_LIST_INFO tsoInfo;
 832         UINT32 packetLength;
 833
 834         tsoInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpLargeSendNetBufferListInfo);
 835         nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 836         packetLength = NET_BUFFER_DATA_LENGTH(nb);
 837
 838         OVS_LOG_TRACE("MSS %u packet len %u",
 839                 tsoInfo.LsoV1Transmit.MSS, packetLength);
 840         if (tsoInfo.LsoV1Transmit.MSS) {
 841             OVS_LOG_TRACE("l4Offset %d", hdrInfo->l4Offset);
 842             newNbl = OvsTcpSegmentNBL(switchContext, nbl, hdrInfo,
 843                     tsoInfo.LsoV1Transmit.MSS , 0);
 844             if (newNbl == NULL) {
 845                 return NDIS_STATUS_FAILURE;
 846             }
 847             nbl = newNbl;
 848         }
 849     }
 850
 851     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 852     while (nb) {
 853         elem = OvsCreateQueuePacket(queueId, userData, userDataLen,
 854                                     cmd, inPort, tunnelKey, nbl, nb,
 855                                     isRecv, hdrInfo);
 856         if (elem) {
 857             InsertTailList(list, &elem->link);
 858             (*num)++;
 859         }
 860         nb = NET_BUFFER_NEXT_NB(nb);
 861     }
 862     if (newNbl) {
 863         OvsCompleteNBL(switchContext, newNbl, TRUE);
 864     }
 865     return NDIS_STATUS_SUCCESS;
 866 }