drivers/staging/lustre/lustre/lov/lov_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2015, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * Implementation of cl_device and cl_device_type for LOV layer.
  37  *
  38  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LOV
  42
  43 /* class_name2obd() */
  44 #include "../include/obd_class.h"
  45
  46 #include "lov_cl_internal.h"
  47 #include "lov_internal.h"
  48
  49 struct kmem_cache *lov_lock_kmem;
  50 struct kmem_cache *lov_object_kmem;
  51 struct kmem_cache *lov_thread_kmem;
  52 struct kmem_cache *lov_session_kmem;
  53 struct kmem_cache *lov_req_kmem;
  54
  55 struct kmem_cache *lovsub_lock_kmem;
  56 struct kmem_cache *lovsub_object_kmem;
  57 struct kmem_cache *lovsub_req_kmem;
  58
  59 struct kmem_cache *lov_lock_link_kmem;
  60
  61 /** Lock class of lov_device::ld_mutex. */
  62 static struct lock_class_key cl_lov_device_mutex_class;
  63
  64 struct lu_kmem_descr lov_caches[] = {
  65         {
  66                 .ckd_cache = &lov_lock_kmem,
  67                 .ckd_name  = "lov_lock_kmem",
  68                 .ckd_size  = sizeof(struct lov_lock)
  69         },
  70         {
  71                 .ckd_cache = &lov_object_kmem,
  72                 .ckd_name  = "lov_object_kmem",
  73                 .ckd_size  = sizeof(struct lov_object)
  74         },
  75         {
  76                 .ckd_cache = &lov_thread_kmem,
  77                 .ckd_name  = "lov_thread_kmem",
  78                 .ckd_size  = sizeof(struct lov_thread_info)
  79         },
  80         {
  81                 .ckd_cache = &lov_session_kmem,
  82                 .ckd_name  = "lov_session_kmem",
  83                 .ckd_size  = sizeof(struct lov_session)
  84         },
  85         {
  86                 .ckd_cache = &lov_req_kmem,
  87                 .ckd_name  = "lov_req_kmem",
  88                 .ckd_size  = sizeof(struct lov_req)
  89         },
  90         {
  91                 .ckd_cache = &lovsub_lock_kmem,
  92                 .ckd_name  = "lovsub_lock_kmem",
  93                 .ckd_size  = sizeof(struct lovsub_lock)
  94         },
  95         {
  96                 .ckd_cache = &lovsub_object_kmem,
  97                 .ckd_name  = "lovsub_object_kmem",
  98                 .ckd_size  = sizeof(struct lovsub_object)
  99         },
 100         {
 101                 .ckd_cache = &lovsub_req_kmem,
 102                 .ckd_name  = "lovsub_req_kmem",
 103                 .ckd_size  = sizeof(struct lovsub_req)
 104         },
 105         {
 106                 .ckd_cache = &lov_lock_link_kmem,
 107                 .ckd_name  = "lov_lock_link_kmem",
 108                 .ckd_size  = sizeof(struct lov_lock_link)
 109         },
 110         {
 111                 .ckd_cache = NULL
 112         }
 113 };
 114
 115 /*****************************************************************************
 116  *
 117  * Lov transfer operations.
 118  *
 119  */
 120
 121 static void lov_req_completion(const struct lu_env *env,
 122                                const struct cl_req_slice *slice, int ioret)
 123 {
 124         struct lov_req *lr;
 125
 126         lr = cl2lov_req(slice);
 127         kmem_cache_free(lov_req_kmem, lr);
 128 }
 129
 130 static const struct cl_req_operations lov_req_ops = {
 131         .cro_completion = lov_req_completion
 132 };
 133
 134 /*****************************************************************************
 135  *
 136  * Lov device and device type functions.
 137  *
 138  */
 139
 140 static void *lov_key_init(const struct lu_context *ctx,
 141                           struct lu_context_key *key)
 142 {
 143         struct lov_thread_info *info;
 144
 145         info = kmem_cache_zalloc(lov_thread_kmem, GFP_NOFS);
 146         if (!info)
 147                 info = ERR_PTR(-ENOMEM);
 148         return info;
 149 }
 150
 151 static void lov_key_fini(const struct lu_context *ctx,
 152                          struct lu_context_key *key, void *data)
 153 {
 154         struct lov_thread_info *info = data;
 155
 156         kmem_cache_free(lov_thread_kmem, info);
 157 }
 158
 159 struct lu_context_key lov_key = {
 160         .lct_tags = LCT_CL_THREAD,
 161         .lct_init = lov_key_init,
 162         .lct_fini = lov_key_fini
 163 };
 164
 165 static void *lov_session_key_init(const struct lu_context *ctx,
 166                                   struct lu_context_key *key)
 167 {
 168         struct lov_session *info;
 169
 170         info = kmem_cache_zalloc(lov_session_kmem, GFP_NOFS);
 171         if (!info)
 172                 info = ERR_PTR(-ENOMEM);
 173         return info;
 174 }
 175
 176 static void lov_session_key_fini(const struct lu_context *ctx,
 177                                  struct lu_context_key *key, void *data)
 178 {
 179         struct lov_session *info = data;
 180
 181         kmem_cache_free(lov_session_kmem, info);
 182 }
 183
 184 struct lu_context_key lov_session_key = {
 185         .lct_tags = LCT_SESSION,
 186         .lct_init = lov_session_key_init,
 187         .lct_fini = lov_session_key_fini
 188 };
 189
 190 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 191 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 192
 193 static struct lu_device *lov_device_fini(const struct lu_env *env,
 194                                          struct lu_device *d)
 195 {
 196         int i;
 197         struct lov_device *ld = lu2lov_dev(d);
 198
 199         LASSERT(ld->ld_lov);
 200         if (!ld->ld_target)
 201                 return NULL;
 202
 203         lov_foreach_target(ld, i) {
 204                 struct lovsub_device *lsd;
 205
 206                 lsd = ld->ld_target[i];
 207                 if (lsd) {
 208                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 209                         ld->ld_target[i] = NULL;
 210                 }
 211         }
 212         return NULL;
 213 }
 214
 215 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 216                            const char *name, struct lu_device *next)
 217 {
 218         struct lov_device *ld = lu2lov_dev(d);
 219         int i;
 220         int rc = 0;
 221
 222         LASSERT(d->ld_site);
 223         if (!ld->ld_target)
 224                 return rc;
 225
 226         lov_foreach_target(ld, i) {
 227                 struct lovsub_device *lsd;
 228                 struct cl_device     *cl;
 229                 struct lov_tgt_desc  *desc;
 230
 231                 desc = ld->ld_lov->lov_tgts[i];
 232                 if (!desc)
 233                         continue;
 234
 235                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 236                                    desc->ltd_obd->obd_lu_dev);
 237                 if (IS_ERR(cl)) {
 238                         rc = PTR_ERR(cl);
 239                         break;
 240                 }
 241                 lsd = cl2lovsub_dev(cl);
 242                 lsd->acid_idx = i;
 243                 lsd->acid_super = ld;
 244                 ld->ld_target[i] = lsd;
 245         }
 246
 247         if (rc)
 248                 lov_device_fini(env, d);
 249         else
 250                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 251
 252         return rc;
 253 }
 254
 255 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
 256                         struct cl_req *req)
 257 {
 258         struct lov_req *lr;
 259         int result;
 260
 261         lr = kmem_cache_zalloc(lov_req_kmem, GFP_NOFS);
 262         if (lr) {
 263                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
 264                 result = 0;
 265         } else {
 266                 result = -ENOMEM;
 267         }
 268         return result;
 269 }
 270
 271 static const struct cl_device_operations lov_cl_ops = {
 272         .cdo_req_init = lov_req_init
 273 };
 274
 275 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 276 {
 277         int i;
 278
 279         for (i = 0; i < nr; ++i) {
 280                 struct lov_device_emerg *em;
 281
 282                 em = emrg[i];
 283                 if (em) {
 284                         LASSERT(em->emrg_page_list.pl_nr == 0);
 285                         if (em->emrg_env)
 286                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 287                         kfree(em);
 288                 }
 289         }
 290         kfree(emrg);
 291 }
 292
 293 static struct lu_device *lov_device_free(const struct lu_env *env,
 294                                          struct lu_device *d)
 295 {
 296         struct lov_device *ld = lu2lov_dev(d);
 297         const int         nr = ld->ld_target_nr;
 298
 299         cl_device_fini(lu2cl_dev(d));
 300         kfree(ld->ld_target);
 301         if (ld->ld_emrg)
 302                 lov_emerg_free(ld->ld_emrg, nr);
 303         kfree(ld);
 304         return NULL;
 305 }
 306
 307 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 308                               __u32 index)
 309 {
 310         struct lov_device *ld = lu2lov_dev(dev);
 311
 312         if (ld->ld_target[index]) {
 313                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 314                 ld->ld_target[index] = NULL;
 315         }
 316 }
 317
 318 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 319 {
 320         struct lov_device_emerg **emerg;
 321         int i;
 322         int result;
 323
 324         emerg = kcalloc(nr, sizeof(emerg[0]), GFP_NOFS);
 325         if (!emerg)
 326                 return ERR_PTR(-ENOMEM);
 327         for (result = i = 0; i < nr && result == 0; i++) {
 328                 struct lov_device_emerg *em;
 329
 330                 em = kzalloc(sizeof(*em), GFP_NOFS);
 331                 if (em) {
 332                         emerg[i] = em;
 333                         cl_page_list_init(&em->emrg_page_list);
 334                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 335                                                     LCT_REMEMBER | LCT_NOREF);
 336                         if (!IS_ERR(em->emrg_env)) {
 337                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 338                         } else {
 339                                 result = PTR_ERR(em->emrg_env);
 340                                 em->emrg_env = NULL;
 341                         }
 342                 } else {
 343                         result = -ENOMEM;
 344                 }
 345         }
 346         if (result != 0) {
 347                 lov_emerg_free(emerg, nr);
 348                 emerg = ERR_PTR(result);
 349         }
 350         return emerg;
 351 }
 352
 353 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 354 {
 355         int   result;
 356         __u32 tgt_size;
 357         __u32 sub_size;
 358
 359         result = 0;
 360         tgt_size = dev->ld_lov->lov_tgt_size;
 361         sub_size = dev->ld_target_nr;
 362         if (sub_size < tgt_size) {
 363                 struct lovsub_device    **newd;
 364                 struct lov_device_emerg **emerg;
 365                 const size_t          sz   = sizeof(newd[0]);
 366
 367                 emerg = lov_emerg_alloc(tgt_size);
 368                 if (IS_ERR(emerg))
 369                         return PTR_ERR(emerg);
 370
 371                 newd = kcalloc(tgt_size, sz, GFP_NOFS);
 372                 if (newd) {
 373                         mutex_lock(&dev->ld_mutex);
 374                         if (sub_size > 0) {
 375                                 memcpy(newd, dev->ld_target, sub_size * sz);
 376                                 kfree(dev->ld_target);
 377                         }
 378                         dev->ld_target    = newd;
 379                         dev->ld_target_nr = tgt_size;
 380
 381                         if (dev->ld_emrg)
 382                                 lov_emerg_free(dev->ld_emrg, sub_size);
 383                         dev->ld_emrg = emerg;
 384                         mutex_unlock(&dev->ld_mutex);
 385                 } else {
 386                         lov_emerg_free(emerg, tgt_size);
 387                         result = -ENOMEM;
 388                 }
 389         }
 390         return result;
 391 }
 392
 393 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 394                              __u32 index)
 395 {
 396         struct obd_device    *obd = dev->ld_obd;
 397         struct lov_device    *ld  = lu2lov_dev(dev);
 398         struct lov_tgt_desc  *tgt;
 399         struct lovsub_device *lsd;
 400         struct cl_device     *cl;
 401         int rc;
 402
 403         obd_getref(obd);
 404
 405         tgt = obd->u.lov.lov_tgts[index];
 406
 407         if (!tgt->ltd_obd->obd_set_up) {
 408                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 409                 return -EINVAL;
 410         }
 411
 412         rc = lov_expand_targets(env, ld);
 413         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 414                 LASSERT(dev->ld_site);
 415
 416                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 417                                    tgt->ltd_obd->obd_lu_dev);
 418                 if (!IS_ERR(cl)) {
 419                         lsd = cl2lovsub_dev(cl);
 420                         lsd->acid_idx = index;
 421                         lsd->acid_super = ld;
 422                         ld->ld_target[index] = lsd;
 423                 } else {
 424                         CERROR("add failed (%d), deleting %s\n", rc,
 425                                obd_uuid2str(&tgt->ltd_uuid));
 426                         lov_cl_del_target(env, dev, index);
 427                         rc = PTR_ERR(cl);
 428                 }
 429         }
 430         obd_putref(obd);
 431         return rc;
 432 }
 433
 434 static int lov_process_config(const struct lu_env *env,
 435                               struct lu_device *d, struct lustre_cfg *cfg)
 436 {
 437         struct obd_device *obd = d->ld_obd;
 438         int cmd;
 439         int rc;
 440         int gen;
 441         __u32 index;
 442
 443         obd_getref(obd);
 444
 445         cmd = cfg->lcfg_command;
 446         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 447         if (rc == 0) {
 448                 switch (cmd) {
 449                 case LCFG_LOV_ADD_OBD:
 450                 case LCFG_LOV_ADD_INA:
 451                         rc = lov_cl_add_target(env, d, index);
 452                         if (rc != 0)
 453                                 lov_del_target(d->ld_obd, index, NULL, 0);
 454                         break;
 455                 case LCFG_LOV_DEL_OBD:
 456                         lov_cl_del_target(env, d, index);
 457                         break;
 458                 }
 459         }
 460         obd_putref(obd);
 461         return rc;
 462 }
 463
 464 static const struct lu_device_operations lov_lu_ops = {
 465         .ldo_object_alloc      = lov_object_alloc,
 466         .ldo_process_config    = lov_process_config,
 467 };
 468
 469 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 470                                           struct lu_device_type *t,
 471                                           struct lustre_cfg *cfg)
 472 {
 473         struct lu_device *d;
 474         struct lov_device *ld;
 475         struct obd_device *obd;
 476         int rc;
 477
 478         ld = kzalloc(sizeof(*ld), GFP_NOFS);
 479         if (!ld)
 480                 return ERR_PTR(-ENOMEM);
 481
 482         cl_device_init(&ld->ld_cl, t);
 483         d = lov2lu_dev(ld);
 484         d->ld_ops       = &lov_lu_ops;
 485         ld->ld_cl.cd_ops = &lov_cl_ops;
 486
 487         mutex_init(&ld->ld_mutex);
 488         lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 489
 490         /* setup the LOV OBD */
 491         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 492         LASSERT(obd);
 493         rc = lov_setup(obd, cfg);
 494         if (rc) {
 495                 lov_device_free(env, d);
 496                 return ERR_PTR(rc);
 497         }
 498
 499         ld->ld_lov = &obd->u.lov;
 500         return d;
 501 }
 502
 503 static const struct lu_device_type_operations lov_device_type_ops = {
 504         .ldto_init = lov_type_init,
 505         .ldto_fini = lov_type_fini,
 506
 507         .ldto_start = lov_type_start,
 508         .ldto_stop  = lov_type_stop,
 509
 510         .ldto_device_alloc = lov_device_alloc,
 511         .ldto_device_free  = lov_device_free,
 512
 513         .ldto_device_init    = lov_device_init,
 514         .ldto_device_fini    = lov_device_fini
 515 };
 516
 517 struct lu_device_type lov_device_type = {
 518         .ldt_tags     = LU_DEVICE_CL,
 519         .ldt_name     = LUSTRE_LOV_NAME,
 520         .ldt_ops      = &lov_device_type_ops,
 521         .ldt_ctx_tags = LCT_CL_THREAD
 522 };
 523 EXPORT_SYMBOL(lov_device_type);
 524
 525 /** @} lov */