datapath-windows/ovsext/User.c

   1 /*
   2  * Copyright (c) 2014 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * OvsUser.c
  19  *      Manage packet queue for packet miss for userAction.
  20  */
  21
  22
  23 #include "precomp.h"
  24
  25 #include "Switch.h"
  26 #include "Vport.h"
  27 #include "Event.h"
  28 #include "User.h"
  29 #include "Datapath.h"
  30 #include "PacketIO.h"
  31 #include "Checksum.h"
  32 #include "NetProto.h"
  33 #include "Flow.h"
  34 #include "TunnelIntf.h"
  35 #include "Jhash.h"
  36
  37 #ifdef OVS_DBG_MOD
  38 #undef OVS_DBG_MOD
  39 #endif
  40 #define OVS_DBG_MOD OVS_DBG_USER
  41 #include "Debug.h"
  42
  43 POVS_PACKET_QUEUE_ELEM OvsGetNextPacket(POVS_OPEN_INSTANCE instance);
  44 extern PNDIS_SPIN_LOCK gOvsCtrlLock;
  45 extern POVS_SWITCH_CONTEXT gOvsSwitchContext;
  46 OVS_USER_STATS ovsUserStats;
  47
  48 static VOID _MapNlAttrToOvsPktExec(PNL_ATTR *nlAttrs, PNL_ATTR *keyAttrs,
  49                                    OvsPacketExecute  *execute);
  50 extern NL_POLICY nlFlowKeyPolicy[];
  51
  52 static __inline VOID
  53 OvsAcquirePidHashLock()
  54 {
  55     NdisAcquireSpinLock(&(gOvsSwitchContext->pidHashLock));
  56 }
  57
  58 static __inline VOID
  59 OvsReleasePidHashLock()
  60 {
  61     NdisReleaseSpinLock(&(gOvsSwitchContext->pidHashLock));
  62 }
  63
  64
  65 static VOID
  66 OvsPurgePacketQueue(POVS_USER_PACKET_QUEUE queue,
  67                     POVS_OPEN_INSTANCE instance)
  68 {
  69     PLIST_ENTRY link, next;
  70     LIST_ENTRY tmp;
  71     POVS_PACKET_QUEUE_ELEM elem;
  72
  73     InitializeListHead(&tmp);
  74     NdisAcquireSpinLock(&queue->queueLock);
  75     if (queue->instance != instance) {
  76         NdisReleaseSpinLock(&queue->queueLock);
  77         return;
  78     }
  79
  80     if (queue->numPackets) {
  81         OvsAppendList(&tmp, &queue->packetList);
  82         queue->numPackets = 0;
  83     }
  84     NdisReleaseSpinLock(&queue->queueLock);
  85     LIST_FORALL_SAFE(&tmp, link, next) {
  86         RemoveEntryList(link);
  87         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
  88         OvsFreeMemory(elem);
  89     }
  90 }
  91
  92 VOID
  93 OvsCleanupPacketQueue(POVS_OPEN_INSTANCE instance)
  94 {
  95     POVS_USER_PACKET_QUEUE queue;
  96     POVS_PACKET_QUEUE_ELEM elem;
  97     PLIST_ENTRY link, next;
  98     LIST_ENTRY tmp;
  99     PIRP irp = NULL;
 100
 101     ASSERT(instance);
 102     InitializeListHead(&tmp);
 103     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 104     if (queue) {
 105         PDRIVER_CANCEL cancelRoutine;
 106         NdisAcquireSpinLock(&queue->queueLock);
 107         ASSERT(queue->instance == instance);
 108         /* XXX Should not happen */
 109         if (queue->instance != instance) {
 110             NdisReleaseSpinLock(&queue->queueLock);
 111             NdisFreeSpinLock(&queue->queueLock);
 112             return;
 113         }
 114
 115         if (queue->numPackets) {
 116             OvsAppendList(&tmp, &queue->packetList);
 117             queue->numPackets = 0;
 118         }
 119         queue->instance = NULL;
 120         instance->packetQueue = NULL;
 121         irp = queue->pendingIrp;
 122         queue->pendingIrp = NULL;
 123         if (irp) {
 124             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 125             if (cancelRoutine == NULL) {
 126                 irp = NULL;
 127             }
 128         }
 129         NdisReleaseSpinLock(&queue->queueLock);
 130         NdisFreeSpinLock(&queue->queueLock);
 131     }
 132     LIST_FORALL_SAFE(&tmp, link, next) {
 133         RemoveEntryList(link);
 134         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 135         OvsFreeMemory(elem);
 136     }
 137     if (irp) {
 138         OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 139     }
 140     if (queue) {
 141         OvsFreeMemory(queue);
 142     }
 143
 144     /* Remove the instance from pidHashArray */
 145     OvsAcquirePidHashLock();
 146     OvsDelPidInstance(gOvsSwitchContext, instance->pid);
 147     OvsReleasePidHashLock();
 148 }
 149
 150 NTSTATUS
 151 OvsSubscribeDpIoctl(PVOID instanceP,
 152                     UINT32 pid,
 153                     UINT8 join)
 154 {
 155     POVS_USER_PACKET_QUEUE queue;
 156     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)instanceP;
 157
 158     OvsAcquireCtrlLock();
 159     if (!gOvsSwitchContext) {
 160         OvsReleaseCtrlLock();
 161         return STATUS_INVALID_PARAMETER;
 162     }
 163     OvsReleaseCtrlLock();
 164
 165     if (instance->packetQueue && !join) {
 166         /* unsubscribe */
 167         OvsCleanupPacketQueue(instance);
 168
 169         OvsAcquirePidHashLock();
 170         /* Remove the instance from pidHashArray */
 171         OvsDelPidInstance(gOvsSwitchContext, pid);
 172         OvsReleasePidHashLock();
 173
 174     } else if (instance->packetQueue == NULL && join) {
 175         queue = (POVS_USER_PACKET_QUEUE) OvsAllocateMemory(sizeof *queue);
 176         if (queue == NULL) {
 177             return STATUS_NO_MEMORY;
 178         }
 179         InitializeListHead(&(instance->pidLink));
 180         instance->packetQueue = queue;
 181         RtlZeroMemory(queue, sizeof (*queue));
 182         NdisAllocateSpinLock(&queue->queueLock);
 183         NdisAcquireSpinLock(&queue->queueLock);
 184         InitializeListHead(&queue->packetList);
 185         queue->pid = pid;
 186         queue->instance = instance;
 187         instance->packetQueue = queue;
 188         NdisReleaseSpinLock(&queue->queueLock);
 189
 190         OvsAcquirePidHashLock();
 191         /* Insert the instance to pidHashArray */
 192         OvsAddPidInstance(gOvsSwitchContext, pid, instance);
 193         OvsReleasePidHashLock();
 194
 195     } else {
 196         /* user mode should call only once for subscribe */
 197         return STATUS_INVALID_PARAMETER;
 198     }
 199
 200     return STATUS_SUCCESS;
 201 }
 202
 203
 204 NTSTATUS
 205 OvsReadDpIoctl(PFILE_OBJECT fileObject,
 206                PVOID outputBuffer,
 207                UINT32 outputLength,
 208                UINT32 *replyLen)
 209 {
 210     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 211     POVS_PACKET_QUEUE_ELEM elem;
 212     UINT32 len;
 213
 214 #define TCP_CSUM_OFFSET  16
 215 #define UDP_CSUM_OFFSET  6
 216     ASSERT(instance);
 217
 218     if (instance->packetQueue == NULL) {
 219         return STATUS_INVALID_PARAMETER;
 220     }
 221     if (outputLength < (sizeof (OVS_PACKET_INFO) + OVS_MIN_PACKET_SIZE)) {
 222         return STATUS_BUFFER_TOO_SMALL;
 223     }
 224
 225     elem = OvsGetNextPacket(instance);
 226     if (elem) {
 227         /*
 228          * XXX revisit this later
 229          */
 230         len = elem->packet.totalLen > outputLength ? outputLength :
 231                  elem->packet.totalLen;
 232
 233         if ((elem->hdrInfo.tcpCsumNeeded || elem->hdrInfo.udpCsumNeeded) &&
 234             len == elem->packet.totalLen) {
 235             UINT16 sum, *ptr;
 236             UINT16 size = (UINT16)(elem->packet.payload - elem->packet.data +
 237                                   elem->hdrInfo.l4Offset);
 238             RtlCopyMemory(outputBuffer, &elem->packet.data, size);
 239             ASSERT(len - size >= elem->hdrInfo.l4PayLoad);
 240             sum = CopyAndCalculateChecksum((UINT8 *)outputBuffer + size,
 241                                            (UINT8 *)&elem->packet.data + size,
 242                                            elem->hdrInfo.l4PayLoad, 0);
 243             ptr =(UINT16 *)((UINT8 *)outputBuffer + size +
 244                             (elem->hdrInfo.tcpCsumNeeded ?
 245                              TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 246             *ptr = sum;
 247             ovsUserStats.l4Csum++;
 248         } else {
 249             RtlCopyMemory(outputBuffer, &elem->packet.data, len);
 250         }
 251
 252         *replyLen = len;
 253         OvsFreeMemory(elem);
 254     }
 255     return STATUS_SUCCESS;
 256 }
 257
 258 /* Helper function to allocate a Forwarding Context for an NBL */
 259 NTSTATUS
 260 OvsAllocateForwardingContextForNBL(POVS_SWITCH_CONTEXT switchContext,
 261                                    PNET_BUFFER_LIST nbl)
 262 {
 263     return switchContext->NdisSwitchHandlers.
 264         AllocateNetBufferListForwardingContext(
 265             switchContext->NdisSwitchContext, nbl);
 266 }
 267
 268 /*
 269  * --------------------------------------------------------------------------
 270  * This function allocates all the stuff necessary for creating an NBL from the
 271  * input buffer of specified length, namely, a nonpaged data buffer of size
 272  * length, an MDL from it, and a NB and NBL from it. It does not allocate an NBL
 273  * context yet. It also copies data from the specified buffer to the NBL.
 274  * --------------------------------------------------------------------------
 275  */
 276 PNET_BUFFER_LIST
 277 OvsAllocateNBLForUserBuffer(POVS_SWITCH_CONTEXT switchContext,
 278                             PVOID userBuffer,
 279                             ULONG length)
 280 {
 281     UINT8 *data = NULL;
 282     PNET_BUFFER_LIST nbl = NULL;
 283     PNET_BUFFER nb;
 284     PMDL mdl;
 285
 286     if (length > OVS_DEFAULT_DATA_SIZE) {
 287         nbl = OvsAllocateVariableSizeNBL(switchContext, length,
 288                                          OVS_DEFAULT_HEADROOM_SIZE);
 289
 290     } else {
 291         nbl = OvsAllocateFixSizeNBL(switchContext, length,
 292                                     OVS_DEFAULT_HEADROOM_SIZE);
 293     }
 294     if (nbl == NULL) {
 295         return NULL;
 296     }
 297
 298     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 299     mdl = NET_BUFFER_CURRENT_MDL(nb);
 300     data = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority) +
 301                     NET_BUFFER_CURRENT_MDL_OFFSET(nb);
 302     if (!data) {
 303         OvsCompleteNBL(switchContext, nbl, TRUE);
 304         return NULL;
 305     }
 306
 307     NdisMoveMemory(data, userBuffer, length);
 308
 309     return nbl;
 310 }
 311
 312 /*
 313  *----------------------------------------------------------------------------
 314  *  OvsNlExecuteCmdHandler --
 315  *    Handler for OVS_PACKET_CMD_EXECUTE command.
 316  *----------------------------------------------------------------------------
 317  */
 318 NTSTATUS
 319 OvsNlExecuteCmdHandler(POVS_USER_PARAMS_CONTEXT usrParamsCtx,
 320                        UINT32 *replyLen)
 321 {
 322     NTSTATUS status = STATUS_SUCCESS;
 323     POVS_MESSAGE msgIn = (POVS_MESSAGE)usrParamsCtx->inputBuffer;
 324     POVS_MESSAGE msgOut = (POVS_MESSAGE)usrParamsCtx->outputBuffer;
 325     PNL_MSG_HDR nlMsgHdr = &(msgIn->nlMsg);
 326     PGENL_MSG_HDR genlMsgHdr = &(msgIn->genlMsg);
 327     POVS_HDR ovsHdr = &(msgIn->ovsHdr);
 328
 329     PNL_ATTR nlAttrs[__OVS_PACKET_ATTR_MAX];
 330     PNL_ATTR keyAttrs[__OVS_KEY_ATTR_MAX] = {NULL};
 331
 332     UINT32 attrOffset = NLMSG_HDRLEN + GENL_HDRLEN + OVS_HDRLEN;
 333     UINT32 keyAttrOffset = 0;
 334     OvsPacketExecute execute;
 335     NL_ERROR nlError = NL_ERROR_SUCCESS;
 336     NL_BUFFER nlBuf;
 337
 338     static const NL_POLICY nlPktExecPolicy[] = {
 339         [OVS_PACKET_ATTR_PACKET] = {.type = NL_A_UNSPEC, .optional = FALSE},
 340         [OVS_PACKET_ATTR_KEY] = {.type = NL_A_UNSPEC, .optional = FALSE},
 341         [OVS_PACKET_ATTR_ACTIONS] = {.type = NL_A_UNSPEC, .optional = FALSE},
 342         [OVS_PACKET_ATTR_USERDATA] = {.type = NL_A_UNSPEC, .optional = TRUE},
 343         [OVS_PACKET_ATTR_EGRESS_TUN_KEY] = {.type = NL_A_UNSPEC,
 344                                             .optional = TRUE}
 345     };
 346
 347     RtlZeroMemory(&execute, sizeof(OvsPacketExecute));
 348
 349     /* Get all the top level Flow attributes */
 350     if ((NlAttrParse(nlMsgHdr, attrOffset, NlMsgAttrsLen(nlMsgHdr),
 351                      nlPktExecPolicy, nlAttrs, ARRAY_SIZE(nlAttrs)))
 352                      != TRUE) {
 353         OVS_LOG_ERROR("Attr Parsing failed for msg: %p",
 354                        nlMsgHdr);
 355         status = STATUS_UNSUCCESSFUL;
 356         goto done;
 357     }
 358
 359     keyAttrOffset = (UINT32)((PCHAR)nlAttrs[OVS_PACKET_ATTR_KEY] -
 360                     (PCHAR)nlMsgHdr);
 361
 362     /* Get flow keys attributes */
 363     if ((NlAttrParseNested(nlMsgHdr, keyAttrOffset,
 364                            NlAttrLen(nlAttrs[OVS_PACKET_ATTR_KEY]),
 365                            nlFlowKeyPolicy, keyAttrs,
 366                            ARRAY_SIZE(keyAttrs))) != TRUE) {
 367         OVS_LOG_ERROR("Key Attr Parsing failed for msg: %p", nlMsgHdr);
 368         status = STATUS_UNSUCCESSFUL;
 369         goto done;
 370     }
 371
 372     execute.dpNo = ovsHdr->dp_ifindex;
 373
 374     _MapNlAttrToOvsPktExec(nlAttrs, keyAttrs, &execute);
 375
 376     status = OvsExecuteDpIoctl(&execute);
 377
 378     /* Default reply that we want to send */
 379     if (status == STATUS_SUCCESS) {
 380         NlBufInit(&nlBuf, usrParamsCtx->outputBuffer,
 381                   usrParamsCtx->outputLength);
 382
 383         /* Prepare nl Msg headers */
 384         status = NlFillOvsMsg(&nlBuf, nlMsgHdr->nlmsgType, 0,
 385                  nlMsgHdr->nlmsgSeq, nlMsgHdr->nlmsgPid,
 386                  genlMsgHdr->cmd, OVS_PACKET_VERSION,
 387                  ovsHdr->dp_ifindex);
 388
 389         if (status == STATUS_SUCCESS) {
 390             *replyLen = msgOut->nlMsg.nlmsgLen;
 391         }
 392     } else {
 393         /* Map NTSTATUS to NL_ERROR */
 394         nlError = NlMapStatusToNlErr(status);
 395
 396         /* As of now there are no transactional errors in the implementation.
 397          * Once we have them then we need to map status to correct
 398          * nlError value, so that below mentioned code gets hit. */
 399         if ((nlError != NL_ERROR_SUCCESS) &&
 400             (usrParamsCtx->outputBuffer)) {
 401
 402             POVS_MESSAGE_ERROR msgError = (POVS_MESSAGE_ERROR)
 403                                            usrParamsCtx->outputBuffer;
 404             BuildErrorMsg(msgIn, msgError, nlError);
 405             *replyLen = msgError->nlMsg.nlmsgLen;
 406             status = STATUS_SUCCESS;
 407             goto done;
 408         }
 409     }
 410
 411 done:
 412     return status;
 413 }
 414
 415 /*
 416  *----------------------------------------------------------------------------
 417  *  _MapNlAttrToOvsPktExec --
 418  *    Maps input Netlink attributes to OvsPacketExecute.
 419  *----------------------------------------------------------------------------
 420  */
 421 static VOID
 422 _MapNlAttrToOvsPktExec(PNL_ATTR *nlAttrs, PNL_ATTR *keyAttrs,
 423                        OvsPacketExecute *execute)
 424 {
 425     execute->packetBuf = NlAttrGet(nlAttrs[OVS_PACKET_ATTR_PACKET]);
 426     execute->packetLen = NlAttrGetSize(nlAttrs[OVS_PACKET_ATTR_PACKET]);
 427
 428     execute->actions = NlAttrGet(nlAttrs[OVS_PACKET_ATTR_ACTIONS]);
 429     execute->actionsLen = NlAttrGetSize(nlAttrs[OVS_PACKET_ATTR_ACTIONS]);
 430
 431     execute->inPort = NlAttrGetU32(keyAttrs[OVS_KEY_ATTR_IN_PORT]);
 432 }
 433
 434 NTSTATUS
 435 OvsExecuteDpIoctl(OvsPacketExecute *execute)
 436 {
 437     NTSTATUS                    status = STATUS_SUCCESS;
 438     NTSTATUS                    ndisStatus;
 439     LOCK_STATE_EX               lockState;
 440     PNET_BUFFER_LIST pNbl;
 441     PNL_ATTR actions;
 442     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 443     OvsFlowKey key;
 444     OVS_PACKET_HDR_INFO layers;
 445     POVS_VPORT_ENTRY vport;
 446
 447     NdisAcquireSpinLock(gOvsCtrlLock);
 448     if (gOvsSwitchContext == NULL) {
 449         status = STATUS_INVALID_PARAMETER;
 450         goto unlock;
 451     }
 452
 453     if (execute->packetLen == 0) {
 454         status = STATUS_INVALID_PARAMETER;
 455         goto unlock;
 456     }
 457
 458     actions = execute->actions;
 459
 460     ASSERT(actions);
 461
 462     /*
 463      * Allocate the NBL, copy the data from the userspace buffer. Allocate
 464      * also, the forwarding context for the packet.
 465      */
 466     pNbl = OvsAllocateNBLForUserBuffer(gOvsSwitchContext, execute->packetBuf,
 467                                        execute->packetLen);
 468     if (pNbl == NULL) {
 469         status = STATUS_NO_MEMORY;
 470         goto unlock;
 471     }
 472
 473     fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(pNbl);
 474     vport = OvsFindVportByPortNo(gOvsSwitchContext, execute->inPort);
 475     if (vport) {
 476         fwdDetail->SourcePortId = vport->portId;
 477         fwdDetail->SourceNicIndex = vport->nicIndex;
 478     } else {
 479         fwdDetail->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 480         fwdDetail->SourceNicIndex = 0;
 481     }
 482     // XXX: Figure out if any of the other members of fwdDetail need to be set.
 483
 484     ndisStatus = OvsExtractFlow(pNbl, fwdDetail->SourcePortId, &key, &layers,
 485                               NULL);
 486     if (ndisStatus == NDIS_STATUS_SUCCESS) {
 487         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 488         NdisAcquireRWLockRead(gOvsSwitchContext->dispatchLock, &lockState,
 489                               NDIS_RWL_AT_DISPATCH_LEVEL);
 490         ndisStatus = OvsActionsExecute(gOvsSwitchContext, NULL, pNbl,
 491                                        vport ? vport->portNo :
 492                                                OVS_DEFAULT_PORT_NO,
 493                                        NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP,
 494                                        &key, NULL, &layers, actions,
 495                                        execute->actionsLen);
 496         pNbl = NULL;
 497         NdisReleaseRWLock(gOvsSwitchContext->dispatchLock, &lockState);
 498     }
 499     if (ndisStatus != NDIS_STATUS_SUCCESS) {
 500         if (ndisStatus == NDIS_STATUS_NOT_SUPPORTED) {
 501             status = STATUS_NOT_SUPPORTED;
 502         } else {
 503             status = STATUS_UNSUCCESSFUL;
 504         }
 505     }
 506
 507     if (pNbl) {
 508         OvsCompleteNBL(gOvsSwitchContext, pNbl, TRUE);
 509     }
 510 unlock:
 511     NdisReleaseSpinLock(gOvsCtrlLock);
 512     return status;
 513 }
 514
 515
 516 NTSTATUS
 517 OvsPurgeDpIoctl(PFILE_OBJECT fileObject)
 518 {
 519     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 520     POVS_USER_PACKET_QUEUE queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 521
 522     if (queue == NULL) {
 523         return STATUS_INVALID_PARAMETER;
 524     }
 525     OvsPurgePacketQueue(queue, instance);
 526     return STATUS_SUCCESS;
 527 }
 528
 529 VOID
 530 OvsCancelIrpDatapath(PDEVICE_OBJECT deviceObject,
 531                      PIRP irp)
 532 {
 533     PIO_STACK_LOCATION irpSp;
 534     PFILE_OBJECT fileObject;
 535     POVS_OPEN_INSTANCE instance;
 536     POVS_USER_PACKET_QUEUE queue = NULL;
 537
 538     UNREFERENCED_PARAMETER(deviceObject);
 539
 540     IoReleaseCancelSpinLock(irp->CancelIrql);
 541     irpSp = IoGetCurrentIrpStackLocation(irp);
 542     fileObject = irpSp->FileObject;
 543
 544     if (fileObject == NULL) {
 545         goto done;
 546     }
 547     NdisAcquireSpinLock(gOvsCtrlLock);
 548     instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 549     if (instance) {
 550         queue = instance->packetQueue;
 551     }
 552     if (instance == NULL || queue == NULL) {
 553         NdisReleaseSpinLock(gOvsCtrlLock);
 554         goto done;
 555     }
 556     NdisReleaseSpinLock(gOvsCtrlLock);
 557     NdisAcquireSpinLock(&queue->queueLock);
 558     if (queue->pendingIrp == irp) {
 559         queue->pendingIrp = NULL;
 560     }
 561     NdisReleaseSpinLock(&queue->queueLock);
 562 done:
 563     OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 564 }
 565
 566
 567 NTSTATUS
 568 OvsWaitDpIoctl(PIRP irp, PFILE_OBJECT fileObject)
 569 {
 570     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 571     POVS_USER_PACKET_QUEUE queue =
 572                (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 573     NTSTATUS status = STATUS_SUCCESS;
 574     BOOLEAN cancelled = FALSE;
 575
 576     if (queue == NULL) {
 577         return STATUS_INVALID_PARAMETER;
 578     }
 579     NdisAcquireSpinLock(&queue->queueLock);
 580     if (queue->instance != instance) {
 581         NdisReleaseSpinLock(&queue->queueLock);
 582         return STATUS_INVALID_PARAMETER;
 583     }
 584     if (queue->pendingIrp) {
 585         NdisReleaseSpinLock(&queue->queueLock);
 586         return STATUS_DEVICE_BUSY;
 587     }
 588     if (queue->numPackets == 0) {
 589         PDRIVER_CANCEL cancelRoutine;
 590         IoMarkIrpPending(irp);
 591         IoSetCancelRoutine(irp, OvsCancelIrpDatapath);
 592         if (irp->Cancel) {
 593             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 594             if (cancelRoutine) {
 595                 cancelled = TRUE;
 596             }
 597         } else {
 598             queue->pendingIrp = irp;
 599         }
 600         status = STATUS_PENDING;
 601     }
 602     NdisReleaseSpinLock(&queue->queueLock);
 603     if (cancelled) {
 604         OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 605         OVS_LOG_INFO("Datapath IRP cancelled: %p", irp);
 606     }
 607     return status;
 608 }
 609
 610
 611 POVS_PACKET_QUEUE_ELEM
 612 OvsGetNextPacket(POVS_OPEN_INSTANCE instance)
 613 {
 614     POVS_USER_PACKET_QUEUE queue;
 615     PLIST_ENTRY link;
 616     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 617     if (queue == NULL) {
 618         return NULL;
 619     }
 620     NdisAcquireSpinLock(&queue->queueLock);
 621     if (queue->instance != instance || queue->numPackets == 0) {
 622         NdisReleaseSpinLock(&queue->queueLock);
 623         return NULL;
 624     }
 625     link = RemoveHeadList(&queue->packetList);
 626     queue->numPackets--;
 627     NdisReleaseSpinLock(&queue->queueLock);
 628     return CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 629 }
 630
 631 /*
 632  * ---------------------------------------------------------------------------
 633  * Given a pid, returns the corresponding USER_PACKET_QUEUE.
 634  * gOvsCtrlLock must be acquired before calling this API.
 635  * ---------------------------------------------------------------------------
 636  */
 637 POVS_USER_PACKET_QUEUE
 638 OvsGetQueue(UINT32 pid)
 639 {
 640     POVS_OPEN_INSTANCE instance;
 641     POVS_USER_PACKET_QUEUE ret = NULL;
 642
 643     instance = OvsGetPidInstance(gOvsSwitchContext, pid);
 644
 645     if (instance) {
 646         ret = instance->packetQueue;
 647     }
 648
 649     return ret;
 650 }
 651
 652 /*
 653  * ---------------------------------------------------------------------------
 654  * Given a pid, returns the corresponding instance.
 655  * pidHashLock must be acquired before calling this API.
 656  * ---------------------------------------------------------------------------
 657  */
 658 POVS_OPEN_INSTANCE
 659 OvsGetPidInstance(POVS_SWITCH_CONTEXT switchContext, UINT32 pid)
 660 {
 661     POVS_OPEN_INSTANCE instance;
 662     PLIST_ENTRY head, link;
 663     UINT32 hash = OvsJhashBytes((const VOID *)&pid, sizeof(pid),
 664                                 OVS_HASH_BASIS);
 665     head = &(switchContext->pidHashArray[hash & OVS_PID_MASK]);
 666     LIST_FORALL(head, link) {
 667         instance = CONTAINING_RECORD(link, OVS_OPEN_INSTANCE, pidLink);
 668         if (instance->pid == pid) {
 669             return instance;
 670         }
 671     }
 672     return NULL;
 673 }
 674
 675 /*
 676  * ---------------------------------------------------------------------------
 677  * Given a pid and an instance. This API adds instance to pidHashArray.
 678  * pidHashLock must be acquired before calling this API.
 679  * ---------------------------------------------------------------------------
 680  */
 681 VOID
 682 OvsAddPidInstance(POVS_SWITCH_CONTEXT switchContext, UINT32 pid,
 683                   POVS_OPEN_INSTANCE instance)
 684 {
 685     PLIST_ENTRY head;
 686     UINT32 hash = OvsJhashBytes((const VOID *)&pid, sizeof(pid),
 687                                 OVS_HASH_BASIS);
 688     head = &(switchContext->pidHashArray[hash & OVS_PID_MASK]);
 689     InsertHeadList(head, &(instance->pidLink));
 690 }
 691
 692 /*
 693  * ---------------------------------------------------------------------------
 694  * Given a pid and an instance. This API removes instance from pidHashArray.
 695  * pidHashLock must be acquired before calling this API.
 696  * ---------------------------------------------------------------------------
 697  */
 698 VOID
 699 OvsDelPidInstance(POVS_SWITCH_CONTEXT switchContext, UINT32 pid)
 700 {
 701     POVS_OPEN_INSTANCE instance = OvsGetPidInstance(switchContext, pid);
 702
 703     if (instance) {
 704         RemoveEntryList(&(instance->pidLink));
 705     }
 706 }
 707
 708 VOID
 709 OvsQueuePackets(PLIST_ENTRY packetList,
 710                 UINT32 numElems)
 711 {
 712     POVS_USER_PACKET_QUEUE upcallQueue = NULL;
 713     POVS_PACKET_QUEUE_ELEM elem;
 714     PIRP irp = NULL;
 715     PLIST_ENTRY  link;
 716     UINT32 num = 0;
 717     LIST_ENTRY dropPackets;
 718
 719     OVS_LOG_LOUD("Enter: numELems: %u", numElems);
 720
 721     InitializeListHead(&dropPackets);
 722
 723     while (!IsListEmpty(packetList)) {
 724         link = RemoveHeadList(packetList);
 725         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 726
 727         ASSERT(elem);
 728
 729         OvsAcquirePidHashLock();
 730
 731         upcallQueue = OvsGetQueue(elem->upcallPid);
 732         if (!upcallQueue) {
 733             /* No upcall queue found, drop this packet. */
 734             InsertTailList(&dropPackets, &elem->link);
 735         } else {
 736             NdisAcquireSpinLock(&upcallQueue->queueLock);
 737
 738             if (upcallQueue->instance == NULL) {
 739                 InsertTailList(&dropPackets, &elem->link);
 740             } else {
 741                 InsertTailList(&upcallQueue->packetList, &elem->link);
 742                 upcallQueue->numPackets++;
 743                 if (upcallQueue->pendingIrp) {
 744                     PDRIVER_CANCEL cancelRoutine;
 745                     irp = upcallQueue->pendingIrp;
 746                     upcallQueue->pendingIrp = NULL;
 747                     cancelRoutine = IoSetCancelRoutine(irp, NULL);
 748                     if (cancelRoutine == NULL) {
 749                         irp = NULL;
 750                     }
 751                 }
 752             }
 753
 754             if (irp) {
 755                 OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 756             }
 757
 758             NdisReleaseSpinLock(&upcallQueue->queueLock);
 759         }
 760
 761         OvsReleasePidHashLock();
 762     }
 763
 764     while (!IsListEmpty(&dropPackets)) {
 765         link = RemoveHeadList(&dropPackets);
 766         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 767         OvsFreeMemory(elem);
 768         num++;
 769     }
 770
 771     OVS_LOG_LOUD("Exit: drop %u packets", num);
 772 }
 773
 774 /*
 775  *----------------------------------------------------------------------------
 776  * OvsCreateAndAddPackets --
 777  *
 778  *  Create a packet and forwarded to user space.
 779  *
 780  *  This function would fragment packet if needed, and queue
 781  *  each segment to user space.
 782  *----------------------------------------------------------------------------
 783  */
 784 NTSTATUS
 785 OvsCreateAndAddPackets(PVOID userData,
 786                        UINT32 userDataLen,
 787                        UINT32 cmd,
 788                        UINT32 inPort,
 789                        OvsFlowKey *key,
 790                        PNET_BUFFER_LIST nbl,
 791                        BOOLEAN isRecv,
 792                        POVS_PACKET_HDR_INFO hdrInfo,
 793                        POVS_SWITCH_CONTEXT switchContext,
 794                        LIST_ENTRY *list,
 795                        UINT32 *num)
 796 {
 797     POVS_PACKET_QUEUE_ELEM elem;
 798     PNET_BUFFER_LIST newNbl = NULL;
 799     PNET_BUFFER nb;
 800
 801     if (hdrInfo->isTcp) {
 802         NDIS_TCP_LARGE_SEND_OFFLOAD_NET_BUFFER_LIST_INFO tsoInfo;
 803         UINT32 packetLength;
 804
 805         tsoInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpLargeSendNetBufferListInfo);
 806         nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 807         packetLength = NET_BUFFER_DATA_LENGTH(nb);
 808
 809         OVS_LOG_TRACE("MSS %u packet len %u",
 810                 tsoInfo.LsoV1Transmit.MSS, packetLength);
 811         if (tsoInfo.LsoV1Transmit.MSS) {
 812             OVS_LOG_TRACE("l4Offset %d", hdrInfo->l4Offset);
 813             newNbl = OvsTcpSegmentNBL(switchContext, nbl, hdrInfo,
 814                     tsoInfo.LsoV1Transmit.MSS , 0);
 815             if (newNbl == NULL) {
 816                 return NDIS_STATUS_FAILURE;
 817             }
 818             nbl = newNbl;
 819         }
 820     }
 821
 822     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 823     while (nb) {
 824         elem = OvsCreateQueueNlPacket(userData, userDataLen,
 825                                     cmd, inPort, key, nbl, nb,
 826                                     isRecv, hdrInfo);
 827         if (elem) {
 828             InsertTailList(list, &elem->link);
 829             (*num)++;
 830         }
 831         nb = NET_BUFFER_NEXT_NB(nb);
 832     }
 833     if (newNbl) {
 834         OvsCompleteNBL(switchContext, newNbl, TRUE);
 835     }
 836     return NDIS_STATUS_SUCCESS;
 837 }
 838
 839 static __inline UINT32
 840 OvsGetUpcallMsgSize(PVOID userData,
 841                     UINT32 userDataLen,
 842                     OvsIPv4TunnelKey *tunnelKey,
 843                     UINT32 payload)
 844 {
 845     UINT32 size = NLMSG_ALIGN(sizeof(struct ovs_header)) +
 846                   NlAttrSize(payload) +
 847                   NlAttrSize(OvsFlowKeyAttrSize());
 848
 849     /* OVS_PACKET_ATTR_USERDATA */
 850     if (userData) {
 851         size += NlAttrTotalSize(userDataLen);
 852     }
 853     /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
 854     /* Is it included in the the flwo key attr XXX */
 855     if (tunnelKey) {
 856         size += NlAttrTotalSize(OvsTunKeyAttrSize());
 857     }
 858     return size;
 859 }
 860
 861 /*
 862  *----------------------------------------------------------------------------
 863  * This function completes the IP Header csum. record the L4 payload offset and
 864  * if there is a need to calculate the TCP or UDP csum. The actual csum will be
 865  * caluculated simopultaneossly with the copy of the payload to the destination
 866  * buffer when the packet is read.
 867  *----------------------------------------------------------------------------
 868  */
 869 static VOID
 870 OvsCompletePacketHeader(UINT8 *packet,
 871                         BOOLEAN isRecv,
 872                         NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csumInfo,
 873                         POVS_PACKET_HDR_INFO hdrInfoIn,
 874                         POVS_PACKET_HDR_INFO hdrInfoOut)
 875 {
 876     if ((isRecv && csumInfo.Receive.IpChecksumValueInvalid) ||
 877         (!isRecv && csumInfo.Transmit.IsIPv4 &&
 878         csumInfo.Transmit.IpHeaderChecksum)) {
 879         PIPV4_HEADER ipHdr = (PIPV4_HEADER)(packet + hdrInfoOut->l3Offset);
 880         ASSERT(hdrInfoIn->isIPv4);
 881         ASSERT(ipHdr->Version == 4);
 882         ipHdr->HeaderChecksum = IPChecksum((UINT8 *)ipHdr,
 883             ipHdr->HeaderLength << 2,
 884             (UINT16)~ipHdr->HeaderChecksum);
 885         ovsUserStats.ipCsum++;
 886     }
 887     ASSERT(hdrInfoIn->tcpCsumNeeded == 0 && hdrInfoOut->udpCsumNeeded == 0);
 888     /*
 889      * calculate TCP/UDP pseudo checksum
 890      */
 891     if (isRecv && csumInfo.Receive.TcpChecksumValueInvalid) {
 892         /*
 893          * Only this case, we need to reclaculate pseudo checksum
 894          * all other cases, it is assumed the pseudo checksum is
 895          * filled already.
 896          *
 897          */
 898         PTCP_HDR tcpHdr = (PTCP_HDR)(packet + hdrInfoIn->l4Offset);
 899         if (hdrInfoIn->isIPv4) {
 900             PIPV4_HEADER ipHdr = (PIPV4_HEADER)(packet + hdrInfoIn->l3Offset);
 901             hdrInfoOut->l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 902                                     (ipHdr->HeaderLength << 2));
 903             tcpHdr->th_sum = IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 904                                          (UINT32 *)&ipHdr->DestinationAddress,
 905                                          IPPROTO_TCP, hdrInfoOut->l4PayLoad);
 906         } else {
 907             PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(packet + hdrInfoIn->l3Offset);
 908             hdrInfoOut->l4PayLoad =
 909                 (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 910                 hdrInfoIn->l3Offset + sizeof(IPV6_HEADER)-
 911                 hdrInfoIn->l4Offset);
 912             ASSERT(hdrInfoIn->isIPv6);
 913             tcpHdr->th_sum =
 914                 IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 915                 (UINT32 *)&ipv6Hdr->DestinationAddress,
 916                 IPPROTO_TCP, hdrInfoOut->l4PayLoad);
 917         }
 918         hdrInfoOut->tcpCsumNeeded = 1;
 919         ovsUserStats.recalTcpCsum++;
 920     } else if (!isRecv) {
 921         if (csumInfo.Transmit.TcpChecksum) {
 922             hdrInfoOut->tcpCsumNeeded = 1;
 923         } else if (csumInfo.Transmit.UdpChecksum) {
 924             hdrInfoOut->udpCsumNeeded = 1;
 925         }
 926         if (hdrInfoOut->tcpCsumNeeded || hdrInfoOut->udpCsumNeeded) {
 927 #ifdef DBG
 928             UINT16 sum, *ptr;
 929             UINT8 proto =
 930                 hdrInfoOut->tcpCsumNeeded ? IPPROTO_TCP : IPPROTO_UDP;
 931 #endif
 932             if (hdrInfoIn->isIPv4) {
 933                 PIPV4_HEADER ipHdr = (PIPV4_HEADER)(packet + hdrInfoIn->l3Offset);
 934                 hdrInfoOut->l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 935                     (ipHdr->HeaderLength << 2));
 936 #ifdef DBG
 937                 sum = IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 938                     (UINT32 *)&ipHdr->DestinationAddress,
 939                     proto, hdrInfoOut->l4PayLoad);
 940 #endif
 941             } else {
 942                 PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(packet +
 943                     hdrInfoIn->l3Offset);
 944                 hdrInfoOut->l4PayLoad =
 945                     (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 946                     hdrInfoIn->l3Offset + sizeof(IPV6_HEADER)-
 947                     hdrInfoIn->l4Offset);
 948                 ASSERT(hdrInfoIn->isIPv6);
 949 #ifdef DBG
 950                 sum = IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 951                     (UINT32 *)&ipv6Hdr->DestinationAddress,
 952                     proto, hdrInfoOut->l4PayLoad);
 953 #endif
 954             }
 955 #ifdef DBG
 956             ptr = (UINT16 *)(packet + hdrInfoIn->l4Offset +
 957                 (hdrInfoOut->tcpCsumNeeded ?
 958             TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 959             ASSERT(*ptr == sum);
 960 #endif
 961         }
 962     }
 963 }
 964
 965 static NTSTATUS
 966 OvsGetPid(POVS_VPORT_ENTRY vport, PNET_BUFFER nb, UINT32 *pid)
 967 {
 968     UNREFERENCED_PARAMETER(nb);
 969
 970     ASSERT(vport);
 971
 972     /* XXX select a pid from an array of pids using a flow based hash */
 973     *pid = vport->upcallPid;
 974     return STATUS_SUCCESS;
 975 }
 976
 977 /*
 978  *----------------------------------------------------------------------------
 979  * OvsCreateQueueNlPacket --
 980  *
 981  *  Create a packet which will be forwarded to user space.
 982  *
 983  * InputParameter:
 984  *   userData: when cmd is user action, this field contain
 985  *      user action data.
 986  *   userDataLen: as name indicated
 987  *   cmd: either miss or user action
 988  *   inPort: datapath port id from which the packet is received.
 989  *   key: flow Key with a tunnel key if available
 990  *   nbl:  the NET_BUFFER_LIST which contain the packet
 991  *   nb: the packet
 992  *   isRecv: This is used to decide how to interprete the csum info
 993  *   hdrInfo: include hdr info initialized during flow extraction.
 994  *
 995  * Results:
 996  *    NULL if fail to create the packet
 997  *    The packet element otherwise
 998  *----------------------------------------------------------------------------
 999  */
1000 POVS_PACKET_QUEUE_ELEM
1001 OvsCreateQueueNlPacket(PVOID userData,
1002                        UINT32 userDataLen,
1003                        UINT32 cmd,
1004                        UINT32 inPort,
1005                        OvsFlowKey *key,
1006                        PNET_BUFFER_LIST nbl,
1007                        PNET_BUFFER nb,
1008                        BOOLEAN isRecv,
1009                        POVS_PACKET_HDR_INFO hdrInfo)
1010 {
1011 #define VLAN_TAG_SIZE 4
1012     UINT32 allocLen, dataLen, extraLen;
1013     POVS_PACKET_QUEUE_ELEM elem;
1014     UINT8 *src, *dst;
1015     NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csumInfo;
1016     NDIS_NET_BUFFER_LIST_8021Q_INFO vlanInfo;
1017     OvsIPv4TunnelKey *tunnelKey = (OvsIPv4TunnelKey *)&key->tunKey;
1018     UINT32 pid;
1019     UINT32 nlMsgSize;
1020     NL_BUFFER nlBuf;
1021     PNL_MSG_HDR nlMsg;
1022
1023     /* XXX pass vport in the stack rather than portNo */
1024     POVS_VPORT_ENTRY vport =
1025         OvsFindVportByPortNo(gOvsSwitchContext, inPort);
1026
1027     if (vport == NULL){
1028         /* No vport is not fatal. */
1029         return NULL;
1030     }
1031
1032     OvsGetPid(vport, nb, &pid);
1033
1034     if (!pid) {
1035         /*
1036          * There is no userspace queue created yet, so there is no point for
1037          * creating a new packet to be queued.
1038          */
1039         return NULL;
1040     }
1041
1042     csumInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo);
1043
1044     if (isRecv && (csumInfo.Receive.TcpChecksumFailed ||
1045                   (csumInfo.Receive.UdpChecksumFailed && !hdrInfo->udpCsumZero) ||
1046                   csumInfo.Receive.IpChecksumFailed)) {
1047         OVS_LOG_INFO("Packet dropped due to checksum failure.");
1048         ovsUserStats.dropDuetoChecksum++;
1049         return NULL;
1050     }
1051
1052     vlanInfo.Value = NET_BUFFER_LIST_INFO(nbl, Ieee8021QNetBufferListInfo);
1053     extraLen = vlanInfo.TagHeader.VlanId ? VLAN_TAG_SIZE : 0;
1054
1055     dataLen = NET_BUFFER_DATA_LENGTH(nb);
1056
1057     if (NlAttrSize(dataLen) > MAXUINT16) {
1058         return NULL;
1059     }
1060
1061     nlMsgSize = OvsGetUpcallMsgSize(userData, userDataLen, tunnelKey,
1062                                     dataLen + extraLen);
1063
1064     allocLen = sizeof (OVS_PACKET_QUEUE_ELEM) + nlMsgSize;
1065     elem = (POVS_PACKET_QUEUE_ELEM)OvsAllocateMemory(allocLen);
1066     if (elem == NULL) {
1067         ovsUserStats.dropDuetoResource++;
1068         return NULL;
1069     }
1070     elem->hdrInfo.value = hdrInfo->value;
1071     elem->upcallPid = pid;
1072     elem->packet.totalLen = nlMsgSize;
1073     /* XXX remove queueid */
1074     elem->packet.queue = 0;
1075     /* XXX  no need as the length is already in the NL attrib */
1076     elem->packet.userDataLen = userDataLen;
1077     elem->packet.inPort = inPort;
1078     elem->packet.cmd = cmd;
1079     if (cmd == (UINT32)OVS_PACKET_CMD_MISS) {
1080         ovsUserStats.miss++;
1081     } else if (cmd == (UINT32)OVS_PACKET_CMD_ACTION) {
1082         ovsUserStats.action++;
1083     } else {
1084         ASSERT(FALSE);
1085         goto fail;
1086     }
1087     /* XXX Should we have both packetLen and TotalLen*/
1088     elem->packet.packetLen = dataLen + extraLen;
1089
1090     NlBufInit(&nlBuf, (PCHAR)elem->packet.data, nlMsgSize);
1091
1092     /*
1093      * Initialize the OVS header
1094      * Since we are pre allocating memory for the NL buffer
1095      * the attribute settings should not fail
1096      */
1097     if (NlFillOvsMsg(&nlBuf, OVS_WIN_NL_PACKET_FAMILY_ID, 0,
1098                       0, pid, (UINT8)cmd, OVS_PACKET_VERSION,
1099                       gOvsSwitchContext->dpNo) != STATUS_SUCCESS) {
1100         goto fail;
1101     }
1102
1103     if (MapFlowKeyToNlKey(&nlBuf, key, OVS_PACKET_ATTR_KEY,
1104                           OVS_KEY_ATTR_TUNNEL) != STATUS_SUCCESS) {
1105         goto fail;
1106     }
1107
1108     /* XXX must send OVS_PACKET_ATTR_EGRESS_TUN_KEY if set by vswtchd */
1109     if (userData){
1110         if (!NlMsgPutTailUnspec(&nlBuf, OVS_PACKET_ATTR_USERDATA,
1111                                 userData, (UINT16)userDataLen)) {
1112             goto fail;
1113         }
1114     }
1115
1116     /*
1117      * Make space for the payload to be copied and set the attribute
1118      * XXX Uninit set initilizes the buffer with xero, we don't actually need
1119      * that the payload to be initailized
1120      */
1121     dst = (UINT8 *)NlMsgPutTailUnspecUninit(&nlBuf, OVS_PACKET_ATTR_PACKET,
1122                                             (UINT16)(dataLen + extraLen));
1123     if (!dst) {
1124         goto fail;
1125     }
1126
1127     /* Store the payload for csum calculation when packet is read */
1128     elem->packet.payload = dst;
1129     dst += extraLen;
1130
1131     src = NdisGetDataBuffer(nb, dataLen, dst, 1, 0);
1132     if (src == NULL) {
1133         ovsUserStats.dropDuetoResource++;
1134         goto fail;
1135     }    else if (src != dst) {
1136         /* Copy the data from the NDIS buffer to dst. */
1137         RtlCopyMemory(dst, src, dataLen);
1138     }
1139
1140     /* Set csum if was offloaded */
1141     OvsCompletePacketHeader(dst, isRecv, csumInfo, hdrInfo, &elem->hdrInfo);
1142
1143     /*
1144      * Finally insert VLAN tag
1145      */
1146     if (extraLen) {
1147         dst = elem->packet.payload;
1148         src = dst + extraLen;
1149         ((UINT32 *)dst)[0] = ((UINT32 *)src)[0];
1150         ((UINT32 *)dst)[1] = ((UINT32 *)src)[1];
1151         ((UINT32 *)dst)[2] = ((UINT32 *)src)[2];
1152         dst += 12;
1153         ((UINT16 *)dst)[0] = htons(0x8100);
1154         ((UINT16 *)dst)[1] = htons(vlanInfo.TagHeader.VlanId |
1155             (vlanInfo.TagHeader.UserPriority << 13));
1156         elem->hdrInfo.l3Offset += VLAN_TAG_SIZE;
1157         elem->hdrInfo.l4Offset += VLAN_TAG_SIZE;
1158         ovsUserStats.vlanInsert++;
1159     }
1160
1161     nlMsg = (PNL_MSG_HDR)NlBufAt(&nlBuf, 0, 0);
1162     nlMsg->nlmsgLen = NlBufSize(&nlBuf);
1163     /* 'totalLen' should be size of valid data. */
1164     elem->packet.totalLen = nlMsg->nlmsgLen;
1165
1166     return elem;
1167 fail:
1168     OvsFreeMemory(elem);
1169     return NULL;
1170 }